Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add secret pydantic fields (#5291)
jnjpnga772bedfe420
diff --git a/fern/openapi.json b/fern/openapi.jsonindex 9a5d17801..e372aa481 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -17232,6 +17232,46 @@],"nullable": true},+ "value_enc": {+ "oneOf": [+ {+ "type": "string"+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "array",+ "items": {+ "oneOf": [+ {+ "type": "string"+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ }+ ],+ "nullable": true+ }+ },+ {}+ ],+ "nullable": true+ },"agent_id": {"type": "string"}@@ -17443,6 +17483,46 @@],"nullable": true},+ "value_enc": {+ "oneOf": [+ {+ "type": "string"+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "array",+ "items": {+ "oneOf": [+ {+ "type": "string"+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ },+ {+ "type": "string",+ "format": "null",+ "nullable": true+ }+ ],+ "nullable": true+ }+ },+ {}+ ],+ "nullable": true+ },"agent_id": {"type": "string"}@@ -21561,6 +21641,20 @@"title": "Description","description": "An optional description of the environment variable."},+ "value_enc": {+ "anyOf": [+ {+ "type": "string",+ "description": "Encrypted secret value (stored as encrypted string)",+ "nullable": true+ },+ {+ "type": "null"+ }+ ],+ "title": "Value Enc",+ "description": "Encrypted value as Secret object"+ },"agent_id": {"type": "string","title": "Agent Id",@@ -30834,6 +30928,34 @@],"title": "Updated At","description": "The last update timestamp of the provider."+ },+ "api_key_enc": {+ "anyOf": [+ {+ "type": "string",+ "description": "Encrypted secret value (stored as encrypted string)",+ "nullable": true+ },+ {+ "type": "null"+ }+ ],+ "title": "Api Key Enc",+ "description": "Encrypted API key as Secret object"+ },+ "access_key_enc": {+ "anyOf": [+ {+ "type": "string",+ "description": "Encrypted secret value (stored as encrypted string)",+ "nullable": true+ },+ {+ "type": "null"+ }+ ],+ "title": "Access Key Enc",+ "description": "Encrypted access key as Secret object"}},"additionalProperties": false,@@ -32021,6 +32143,20 @@"title": "Description","description": "An optional description of the environment variable."},+ "value_enc": {+ "anyOf": [+ {+ "type": "string",+ "description": "Encrypted secret value (stored as encrypted string)",+ "nullable": true+ },+ {+ "type": "null"+ }+ ],+ "title": "Value Enc",+ "description": "Encrypted value as Secret object"+ },"sandbox_config_id": {"type": "string","title": "Sandbox Config Id",diff --git a/letta/schemas/environment_variables.py b/letta/schemas/environment_variables.pyindex bf423e06e..0a0e53c7a 100644--- a/letta/schemas/environment_variables.py+++ b/letta/schemas/environment_variables.py@@ -3,6 +3,8 @@ from typing import Optionalfrom pydantic import Fieldfrom letta.schemas.letta_base import LettaBase, OrmMetadataBase+from letta.schemas.secret import Secret+from letta.settings import settings# Base Environment Variable@@ -13,6 +15,28 @@ class EnvironmentVariableBase(OrmMetadataBase):description: Optional[str] = Field(None, description="An optional description of the environment variable.")organization_id: Optional[str] = Field(None, description="The ID of the organization this environment variable belongs to.")+ # Encrypted field (stored as Secret object, serialized to string for DB)+ # Secret class handles validation and serialization automatically via __get_pydantic_core_schema__+ value_enc: Secret | None = Field(None, description="Encrypted value as Secret object")++ def get_value_secret(self) -> Secret:+ """Get the value as a Secret object, preferring encrypted over plaintext."""+ # If value_enc is already a Secret, return it+ if self.value_enc is not None:+ return self.value_enc+ # Otherwise, create from plaintext value+ return Secret.from_db(None, self.value)++ def set_value_secret(self, secret: Secret) -> None:+ """Set value from a Secret object, directly storing the Secret."""+ self.value_enc = secret+ # Also update plaintext field for dual-write during migration+ secret_dict = secret.to_dict()+ if not secret.was_encrypted:+ self.value = secret_dict["plaintext"]+ else:+ self.value = None+class EnvironmentVariableCreateBase(LettaBase):key: str = Field(..., description="The name of the environment variable.")diff --git a/letta/schemas/providers/base.py b/letta/schemas/providers/base.pyindex 8187f30a0..a88fd5a7f 100644--- a/letta/schemas/providers/base.py+++ b/letta/schemas/providers/base.py@@ -8,6 +8,7 @@ from letta.schemas.enums import ProviderCategory, ProviderTypefrom letta.schemas.letta_base import LettaBasefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.llm_config_overrides import LLM_HANDLE_OVERRIDES+from letta.schemas.secret import Secretfrom letta.settings import model_settings@@ -28,8 +29,14 @@ class Provider(ProviderBase):organization_id: str | None = Field(None, description="The organization id of the user")updated_at: datetime | None = Field(None, description="The last update timestamp of the provider.")+ # Encrypted fields (stored as Secret objects, serialized to strings for DB)+ # Secret class handles validation and serialization automatically via __get_pydantic_core_schema__+ api_key_enc: Secret | None = Field(None, description="Encrypted API key as Secret object")+ access_key_enc: Secret | None = Field(None, description="Encrypted access key as Secret object")+@model_validator(mode="after")def default_base_url(self):+ # Set default base URLif self.provider_type == ProviderType.openai and self.base_url is None:self.base_url = model_settings.openai_api_basereturn self@@ -38,6 +45,42 @@ class Provider(ProviderBase):if not self.id:self.id = ProviderBase.generate_id(prefix=ProviderBase.__id_prefix__)+ def get_api_key_secret(self) -> Secret:+ """Get the API key as a Secret object, preferring encrypted over plaintext."""+ # If api_key_enc is already a Secret, return it+ if self.api_key_enc is not None:+ return self.api_key_enc+ # Otherwise, create from plaintext api_key+ return Secret.from_db(None, self.api_key)++ def get_access_key_secret(self) -> Secret:+ """Get the access key as a Secret object, preferring encrypted over plaintext."""+ # If access_key_enc is already a Secret, return it+ if self.access_key_enc is not None:+ return self.access_key_enc+ # Otherwise, create from plaintext access_key+ return Secret.from_db(None, self.access_key)++ def set_api_key_secret(self, secret: Secret) -> None:+ """Set API key from a Secret object, directly storing the Secret."""+ self.api_key_enc = secret+ # Also update plaintext field for dual-write during migration+ secret_dict = secret.to_dict()+ if not secret.was_encrypted:+ self.api_key = secret_dict["plaintext"]+ else:+ self.api_key = None++ def set_access_key_secret(self, secret: Secret) -> None:+ """Set access key from a Secret object, directly storing the Secret."""+ self.access_key_enc = secret+ # Also update plaintext field for dual-write during migration+ secret_dict = secret.to_dict()+ if not secret.was_encrypted:+ self.access_key = secret_dict["plaintext"]+ else:+ self.access_key = None+async def check_api_key(self):"""Check if the API key is valid for the provider"""raise NotImplementedErrordiff --git a/letta/schemas/secret.py b/letta/schemas/secret.pyindex 45059aef7..8986a1f51 100644--- a/letta/schemas/secret.py+++ b/letta/schemas/secret.py@@ -2,6 +2,7 @@ import jsonfrom typing import Any, Dict, Optionalfrom pydantic import BaseModel, ConfigDict, PrivateAttr+from pydantic_core import core_schemafrom letta.helpers.crypto_utils import CryptoUtilsfrom letta.log import get_logger@@ -19,16 +20,16 @@ class Secret(BaseModel):TODO: Once we deprecate plaintext columns in the database:- Remove the dual-write logic in to_dict()- Remove the from_db() method's plaintext_value parameter- - Remove the _was_encrypted flag (no longer needed for migration)+ - Remove the was_encrypted flag (no longer needed for migration)- Simplify get_plaintext() to only handle encrypted values"""- # Store the encrypted value- _encrypted_value: Optional[str] = PrivateAttr(default=None)- # Cache the decrypted value to avoid repeated decryption+ # Store the encrypted value as a regular field+ encrypted_value: Optional[str] = None+ # Cache the decrypted value to avoid repeated decryption (not serialized for security)_plaintext_cache: Optional[str] = PrivateAttr(default=None)# Flag to indicate if the value was originally encrypted- _was_encrypted: bool = PrivateAttr(default=False)+ was_encrypted: bool = Falsemodel_config = ConfigDict(frozen=True)@@ -44,18 +45,16 @@ class Secret(BaseModel):A Secret instance with the encrypted value, or plaintext if encryption unavailable"""if value is None:- instance = cls()- instance._encrypted_value = None- instance._was_encrypted = False- return instance+ return cls.model_construct(encrypted_value=None, was_encrypted=False)++ # Guard against double encryption - check if value is already encrypted+ if CryptoUtils.is_encrypted(value):+ logger.warning("Creating Secret from already-encrypted value. This can be dangerous.")# Try to encrypt, but fall back to plaintext if no encryption keytry:encrypted = CryptoUtils.encrypt(value)- instance = cls()- instance._encrypted_value = encrypted- instance._was_encrypted = False- return instance+ return cls.model_construct(encrypted_value=encrypted, was_encrypted=False)except ValueError as e:# No encryption key available, store as plaintextif "No encryption key configured" in str(e):@@ -63,10 +62,8 @@ class Secret(BaseModel):"No encryption key configured. Storing Secret value as plaintext. ""Set LETTA_ENCRYPTION_KEY environment variable to enable encryption.")- instance = cls()- instance._encrypted_value = value # Store plaintext+ instance = cls.model_construct(encrypted_value=value, was_encrypted=False)instance._plaintext_cache = value # Cache it- instance._was_encrypted = Falsereturn instanceraise # Re-raise if it's a different error@@ -81,10 +78,7 @@ class Secret(BaseModel):Returns:A Secret instance"""- instance = cls()- instance._encrypted_value = encrypted_value- instance._was_encrypted = True- return instance+ return cls.model_construct(encrypted_value=encrypted_value, was_encrypted=True)@classmethoddef from_db(cls, encrypted_value: Optional[str], plaintext_value: Optional[str]) -> "Secret":@@ -114,7 +108,7 @@ class Secret(BaseModel):Returns:The encrypted value, or None if the secret is empty"""- return self._encrypted_value+ return self.encrypted_valuedef get_plaintext(self) -> Optional[str]:"""@@ -126,7 +120,7 @@ class Secret(BaseModel):Returns:The decrypted plaintext value"""- if self._encrypted_value is None:+ if self.encrypted_value is None:return None# Use cached value if available, but only if it looks like plaintext@@ -134,14 +128,14 @@ class Secret(BaseModel):if self._plaintext_cache is not None:# If we have a cache but the stored value looks encrypted and we have no key,# we should not use the cache- if CryptoUtils.is_encrypted(self._encrypted_value) and not CryptoUtils.is_encryption_available():+ if CryptoUtils.is_encrypted(self.encrypted_value) and not CryptoUtils.is_encryption_available():self._plaintext_cache = None # Clear invalid cacheelse:return self._plaintext_cache# Decrypt and cachetry:- plaintext = CryptoUtils.decrypt(self._encrypted_value)+ plaintext = CryptoUtils.decrypt(self.encrypted_value)# Cache the decrypted value (PrivateAttr fields can be mutated even with frozen=True)self._plaintext_cache = plaintextreturn plaintext@@ -151,7 +145,7 @@ class Secret(BaseModel):# Handle missing encryption keyif "No encryption key configured" in error_msg:# Check if the value looks encrypted- if CryptoUtils.is_encrypted(self._encrypted_value):+ if CryptoUtils.is_encrypted(self.encrypted_value):# Value was encrypted, but now we have no key - can't decryptlogger.warning("Cannot decrypt Secret value - no encryption key configured. "@@ -162,26 +156,26 @@ class Secret(BaseModel):else:# Value is plaintext (stored when no key was available)logger.debug("Secret value is plaintext (stored without encryption)")- self._plaintext_cache = self._encrypted_value- return self._encrypted_value+ self._plaintext_cache = self.encrypted_value+ return self.encrypted_value# Handle decryption failure (might be plaintext stored as such)elif "Failed to decrypt data" in error_msg:# Check if it might be plaintext- if not CryptoUtils.is_encrypted(self._encrypted_value):+ if not CryptoUtils.is_encrypted(self.encrypted_value):# It's plaintext that was stored when no key was availablelogger.debug("Secret value appears to be plaintext (stored without encryption)")- self._plaintext_cache = self._encrypted_value- return self._encrypted_value+ self._plaintext_cache = self.encrypted_value+ return self.encrypted_value# Otherwise, it's corrupted or wrong keylogger.error("Failed to decrypt Secret value - data may be corrupted or wrong key")raise# Migration case: handle legacy plaintext- elif not self._was_encrypted:- if self._encrypted_value and not CryptoUtils.is_encrypted(self._encrypted_value):- self._plaintext_cache = self._encrypted_value- return self._encrypted_value+ elif not self.was_encrypted:+ if self.encrypted_value and not CryptoUtils.is_encrypted(self.encrypted_value):+ self._plaintext_cache = self.encrypted_value+ return self.encrypted_valuereturn None# Re-raise for other errors@@ -189,7 +183,7 @@ class Secret(BaseModel):def is_empty(self) -> bool:"""Check if the secret is empty/None."""- return self._encrypted_value is None+ return self.encrypted_value is Nonedef __str__(self) -> str:"""String representation that doesn't expose the actual value."""@@ -207,7 +201,7 @@ class Secret(BaseModel):Returns both encrypted and plaintext values for dual-write during migration."""- return {"encrypted": self.get_encrypted(), "plaintext": self.get_plaintext() if not self._was_encrypted else None}+ return {"encrypted": self.get_encrypted(), "plaintext": self.get_plaintext() if not self.was_encrypted else None}def __eq__(self, other: Any) -> bool:"""@@ -219,6 +213,79 @@ class Secret(BaseModel):return Falsereturn self.get_plaintext() == other.get_plaintext()+ @classmethod+ def __get_pydantic_core_schema__(cls, source_type: Any, handler) -> core_schema.CoreSchema:+ """+ Customize Pydantic's validation and serialization behavior for Secret fields.++ This allows Secret fields to automatically:+ - Deserialize: Convert encrypted strings from DB → Secret objects+ - Serialize: Convert Secret objects → encrypted strings for DB+ """++ def validate_secret(value: Any) -> "Secret":+ """Convert various input types to Secret objects."""+ if isinstance(value, Secret):+ return value+ elif isinstance(value, str):+ # String from DB is assumed to be encrypted+ return Secret.from_encrypted(value)+ elif isinstance(value, dict):+ # Dict might be from Pydantic serialization - check for encrypted_value key+ if "encrypted_value" in value:+ # This is a serialized Secret being deserialized+ return cls(**value)+ elif not value or value == {}:+ # Empty dict means None+ return Secret.from_plaintext(None)+ else:+ raise ValueError(f"Cannot convert dict to Secret: {value}")+ elif value is None:+ return Secret.from_plaintext(None)+ else:+ raise ValueError(f"Cannot convert {type(value)} to Secret")++ def serialize_secret(secret: "Secret") -> Optional[str]:+ """Serialize Secret to encrypted string."""+ if secret is None:+ return None+ return secret.get_encrypted()++ python_schema = core_schema.chain_schema(+ [+ core_schema.no_info_plain_validator_function(validate_secret),+ core_schema.is_instance_schema(cls),+ ]+ )++ return core_schema.json_or_python_schema(+ json_schema=python_schema,+ python_schema=python_schema,+ serialization=core_schema.plain_serializer_function_ser_schema(+ serialize_secret,+ when_used="always",+ ),+ )++ @classmethod+ def __get_pydantic_json_schema__(cls, core_schema: core_schema.CoreSchema, handler) -> Dict[str, Any]:+ """+ Define JSON schema representation for Secret fields.+ In JSON schema (OpenAPI docs), Secret fields appear as nullable strings.+ The actual encryption/decryption happens at runtime via __get_pydantic_core_schema__.+ Args:+ core_schema: The core schema for this type+ handler: Handler for generating JSON schema+ Returns:+ A JSON schema dict representing this type as a nullable string+ """+ # Return a simple string schema for JSON schema generation+ return {+ "type": "string",+ "nullable": True,+ "description": "Encrypted secret value (stored as encrypted string)",+ }+class SecretDict(BaseModel):"""