LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2696 [FEAT] Add cloud-only configuration registry for organization-specific settings (#1473)
vishnuszipstackc5a752c57733
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex e7cdc8821..04154dd37 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -2,7 +2,7 @@ import jsonimport loggingfrom typing import Any-from django.conf import settings+from configuration.models import Configurationfrom django.db.models import QuerySetfrom django.http import HttpResponsefrom permissions.permission import IsOwner@@ -110,7 +110,21 @@ class DeploymentExecution(views.APIView):response_status = status.HTTP_422_UNPROCESSABLE_ENTITYif response.execution_status == CeleryTaskState.COMPLETED.value:response_status = status.HTTP_200_OK- if not settings.ENABLE_HIGHLIGHT_API_DEPLOYMENT:+ # Check if highlight data should be removed using configuration registry+ api_deployment = deployment_execution_dto.api+ organization = api_deployment.organization if api_deployment else None+ enable_highlight = False # Safe default if the key is unavailable (e.g., OSS)+ # Check if the configuration key exists (Cloud deployment) or use settings (OSS)+ from configuration.config_registry import ConfigurationRegistry++ if ConfigurationRegistry.is_config_key_available(+ "ENABLE_HIGHLIGHT_API_DEPLOYMENT"+ ):+ enable_highlight = Configuration.get_value_by_organization(+ config_key="ENABLE_HIGHLIGHT_API_DEPLOYMENT",+ organization=organization,+ )+ if not enable_highlight:response.remove_result_metadata_keys(["highlight_data"])if not include_metadata:response.remove_result_metadata_keys()diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 764e22e00..bfaf97a4f 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -2,7 +2,7 @@ import loggingfrom typing import Anyfrom urllib.parse import urlencode-from django.conf import settings+from configuration.models import Configurationfrom django.core.files.uploadedfile import UploadedFilefrom rest_framework.request import Requestfrom rest_framework.serializers import Serializer@@ -204,7 +204,19 @@ class DeploymentHelper(BaseAPIKeyValidator):result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)- if not settings.ENABLE_HIGHLIGHT_API_DEPLOYMENT:+ # Check if highlight data should be removed using configuration registry+ organization = api.organization if api else None+ enable_highlight = False # Safe default if the key is unavailable (e.g., OSS)+ from configuration.config_registry import ConfigurationRegistry++ if ConfigurationRegistry.is_config_key_available(+ "ENABLE_HIGHLIGHT_API_DEPLOYMENT"+ ):+ enable_highlight = Configuration.get_value_by_organization(+ config_key="ENABLE_HIGHLIGHT_API_DEPLOYMENT",+ organization=organization,+ )+ if not enable_highlight:result.remove_result_metadata_keys(["highlight_data"])if not include_metadata:result.remove_result_metadata_keys()diff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.pynew file mode 100644index 000000000..189126631--- /dev/null+++ b/backend/configuration/config_registry.py@@ -0,0 +1,209 @@+"""Configuration registry for dynamically loading configuration keys.++This module handles loading both base OSS configuration keys and+cloud-specific configuration keys from plugins when available.+"""++import logging+import os+from importlib import import_module+from typing import Any++from django.apps import apps++from configuration.constants import ConfigPluginConstants+from configuration.enums import ConfigKey, ConfigSpec++logger = logging.getLogger(__name__)+++def _load_cloud_config_specs() -> dict[str, ConfigSpec]:+ """Load cloud configuration specs from plugins if available.++ Returns:+ Dictionary of cloud configuration specs, empty if plugin not available.+ """+ cloud_specs = {}++ try:+ # Check if plugins app exists+ auth_app = apps.get_app_config(ConfigPluginConstants.PLUGINS_APP)+ config_dir = os.path.join(auth_app.path, ConfigPluginConstants.CONFIG_PLUGIN_DIR)++ # Check if configuration plugin directory exists+ if not os.path.exists(config_dir):+ logger.debug("Configuration plugin directory not found")+ return cloud_specs++ # Try to import the configuration plugin package+ config_package_path = (+ f"{auth_app.module.__package__}.{ConfigPluginConstants.CONFIG_PLUGIN_DIR}"+ )++ try:+ # Import the plugin package to get metadata from __init__.py+ plugin_module = import_module(config_package_path)+ metadata = getattr(plugin_module, ConfigPluginConstants.CONFIG_METADATA, {})++ if metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):+ # Import the cloud_config module to get config specs+ full_module_path = (+ f"{config_package_path}.{ConfigPluginConstants.CONFIG_MODULE}"+ )+ config_module = import_module(full_module_path)+ config_specs = getattr(+ config_module, ConfigPluginConstants.CONFIG_SPECS, {}+ )+ cloud_specs.update(config_specs)+ logger.info(+ "Loaded cloud configuration plugin: %s with %d config keys",+ metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),+ len(config_specs),+ )+ else:+ logger.debug("Cloud configuration plugin not active")++ except ModuleNotFoundError as e:+ logger.debug("Cloud configuration module not found: %s", e)+ except Exception as e:+ logger.error("Error loading cloud configuration plugin: %s", e)++ except LookupError:+ logger.debug("Plugins app not configured")+ except Exception as e:+ logger.error("Error accessing plugins app: %s", e)++ return cloud_specs+++class ConfigurationRegistry:+ """Registry for managing all configuration keys (OSS and cloud).++ This registry provides a unified interface to access both base OSS+ configuration keys and cloud-specific keys when available.+ """++ # Cache for all config specs (loaded once)+ _all_config_specs: dict[str, ConfigSpec] | None = None++ @classmethod+ def _load_all_specs(cls) -> dict[str, ConfigSpec]:+ """Load all configuration specs (OSS + cloud).++ Returns:+ Dictionary mapping config key names to ConfigSpec objects.+ """+ if cls._all_config_specs is not None:+ return cls._all_config_specs++ all_specs = {}++ # First, add all OSS config keys+ for config_key in ConfigKey:+ all_specs[config_key.name] = config_key.value++ # Then, add cloud config keys if available+ cloud_specs = _load_cloud_config_specs()+ all_specs.update(cloud_specs)++ # Cache the result+ cls._all_config_specs = all_specs+ logger.info("Configuration registry loaded with %d total keys", len(all_specs))++ return all_specs++ @classmethod+ def get_config_spec(cls, key_name: str) -> ConfigSpec | None:+ """Get configuration spec for a given key name.++ Args:+ key_name: Name of the configuration key (e.g., "ENABLE_HIGHLIGHT_API_DEPLOYMENT")++ Returns:+ ConfigSpec object if found, None otherwise.+ """+ all_specs = cls._load_all_specs()+ return all_specs.get(key_name)++ @classmethod+ def get_all_config_keys(cls) -> dict[str, ConfigSpec]:+ """Get all available configuration keys.++ Returns:+ Dictionary of all configuration specs (OSS + cloud).+ """+ return cls._load_all_specs().copy()++ @classmethod+ def is_config_key_available(cls, key_name: str) -> bool:+ """Check if a configuration key is available.++ Args:+ key_name: Name of the configuration key.++ Returns:+ True if the key exists, False otherwise.+ """+ return key_name in cls._load_all_specs()++ @classmethod+ def cast_value(cls, key_name: str, raw_value: Any) -> Any:+ """Cast a raw value to the appropriate type for a config key.++ Args:+ key_name: Name of the configuration key.+ raw_value: Raw value to cast.++ Returns:+ Casted value according to the ConfigSpec.++ Raises:+ ValueError: If key not found or casting fails.+ """+ spec = cls.get_config_spec(key_name)+ if not spec:+ raise ValueError(f"Configuration key '{key_name}' not found")++ # Try to use ConfigKey enum's cast_value if it's an OSS key+ try:+ config_key = ConfigKey[key_name]+ return config_key.cast_value(raw_value)+ except KeyError:+ # It's a cloud key, cast it manually+ from configuration.enums import ConfigType++ converters = {+ ConfigType.INT: int,+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1")+ if isinstance(v, str)+ else bool(v),+ ConfigType.JSON: lambda v: __import__("json").loads(v),+ ConfigType.STRING: str,+ }++ converter = converters.get(spec.value_type)+ if not converter:+ raise ValueError(f"Unknown value type: {spec.value_type}")++ try:+ converted_value = converter(raw_value)+ # Validate min/max if applicable+ if (+ hasattr(spec, "min_value")+ and spec.min_value is not None+ and converted_value < spec.min_value+ ):+ raise ValueError(+ f"Value {converted_value} is below minimum {spec.min_value}"+ )+ if (+ hasattr(spec, "max_value")+ and spec.max_value is not None+ and converted_value > spec.max_value+ ):+ raise ValueError(+ f"Value {converted_value} is above maximum {spec.max_value}"+ )+ return converted_value+ except Exception as e:+ raise ValueError(f"Failed to cast value '{raw_value}': {e}")diff --git a/backend/configuration/constants.py b/backend/configuration/constants.pynew file mode 100644index 000000000..524e2c155--- /dev/null+++ b/backend/configuration/constants.py@@ -0,0 +1,13 @@+"""Constants for configuration module."""+++class ConfigPluginConstants:+ """Constants for configuration plugin loading."""++ PLUGINS_APP = "plugins"+ CONFIG_PLUGIN_DIR = "configuration"+ CONFIG_MODULE = "cloud_config"+ CONFIG_METADATA = "metadata"+ CONFIG_SPECS = "config_specs"+ METADATA_IS_ACTIVE = "is_active"+ METADATA_NAME = "name"diff --git a/backend/configuration/enums.py b/backend/configuration/enums.pyindex 369b1010f..f35033a9f 100644--- a/backend/configuration/enums.py+++ b/backend/configuration/enums.py@@ -64,7 +64,9 @@ class ConfigKey(Enum):def cast_value(self, raw_value: Any):converters = {ConfigType.INT: int,- ConfigType.BOOL: lambda v: v.lower() in ("true", "1"),+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1")+ if isinstance(v, str)+ else bool(v),ConfigType.JSON: json.loads,ConfigType.STRING: str,}diff --git a/backend/configuration/models.py b/backend/configuration/models.pyindex 8cd3f803e..1d31213a1 100644--- a/backend/configuration/models.py+++ b/backend/configuration/models.py@@ -7,6 +7,7 @@ from django.core.exceptions import ValidationErrorfrom django.db import modelsfrom utils.models.base_model import BaseModel+from configuration.config_registry import ConfigurationRegistryfrom configuration.enums import ConfigKeylogger = logging.getLogger(__name__)@@ -47,51 +48,65 @@ class Configuration(BaseModel):]def clean(self):- """Validate that the key field contains a valid ConfigKey enum value."""+ """Validate that the key field contains a valid configuration key."""super().clean()- if self.key and self.key not in [config.name for config in ConfigKey]:+ if self.key and not ConfigurationRegistry.is_config_key_available(self.key):+ available_keys = list(ConfigurationRegistry.get_all_config_keys().keys())raise ValidationError({- "key": f"Invalid configuration key '{self.key}'. Must be one of: {[config.name for config in ConfigKey]}"+ "key": f"Invalid configuration key '{self.key}'. Must be one of: {available_keys}"})@propertydef typed_value(self) -> Any:- """Convert stored string value to proper type based on ConfigKey specification."""+ """Convert stored string value to proper type based on configuration specification."""try:- spec = ConfigKey[self.key].cast_value(self.value)- return spec+ return ConfigurationRegistry.cast_value(self.key, self.value)except (ValueError, KeyError, TypeError):return None@classmethoddef get_value_by_organization(- cls, config_key: ConfigKey, organization: Organization | None = None+ cls, config_key: ConfigKey | str, organization: Organization | None = None) -> Any:"""Get configuration value for an organization with proper fallback to defaults.This method handles all error cases and returns the appropriate default value:- - If no organization provided: returns ConfigKey default- - If configuration not found: returns ConfigKey default- - If configuration disabled: returns ConfigKey default- - If value casting fails: returns ConfigKey default- - If value is invalid (e.g., 0, negative): returns ConfigKey default+ - If no organization provided: returns config default+ - If configuration not found: returns config default+ - If configuration disabled: returns config default+ - If value casting fails: returns config default+ - If value is invalid (e.g., 0, negative): returns config defaultArgs:- config_key: The configuration key enum to retrieve+ config_key: The configuration key (ConfigKey enum or string name) to retrieveorganization: The organization to get config for, None for defaultReturns:The typed configuration value or default if not found/invalid"""+ # Handle both ConfigKey enum and string key names+ if isinstance(config_key, ConfigKey):+ key_name = config_key.name+ else:+ key_name = config_key++ # Get the config spec from registry+ config_spec = ConfigurationRegistry.get_config_spec(key_name)+ if not config_spec:+ logger.error(f"Configuration key '{key_name}' not found in registry")+ raise ValueError(f"Unknown configuration key: {key_name}")++ default_value = config_spec.default+if not organization:- return config_key.value.default+ return default_valuetry:- config = cls.objects.get(organization=organization, key=config_key.name)+ config = cls.objects.get(organization=organization, key=key_name)if not config.enabled:- return config_key.value.default+ return default_value# Get the typed value, which includes validationtyped_value = config.typed_value@@ -99,18 +114,18 @@ class Configuration(BaseModel):# Additional safety check - if typed_value is None, return defaultif typed_value is None:logger.warning(- f"Configuration {config_key.name} for organization {organization.id} "- f"returned None, using default value {config_key.value.default}"+ f"Configuration {key_name} for organization {organization.id} "+ f"returned None, using default value {default_value}")- return config_key.value.default+ return default_valuereturn typed_valueexcept cls.DoesNotExist:- return config_key.value.default+ return default_valueexcept Exception as e:logger.warning(- f"Configuration {config_key.name} for organization {organization.id if organization else 'None'} "- f"has invalid value, using default {config_key.value.default}. Error: {e}"+ f"Configuration {key_name} for organization {organization.id if organization else 'None'} "+ f"has invalid value, using default {default_value}. Error: {e}")- return config_key.value.default+ return default_value