Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: add secret pydantic fields (#5291)

jnjpnga772bedfe420
diff --git a/fern/openapi.json b/fern/openapi.json
index 9a5d17801..e372aa481 100644
--- a/fern/openapi.json
+++ b/fern/openapi.json
@@ -17232,6 +17232,46 @@
],
"nullable": true
},
+ "value_enc": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ }
+ ],
+ "nullable": true
+ }
+ },
+ {}
+ ],
+ "nullable": true
+ },
"agent_id": {
"type": "string"
}
@@ -17443,6 +17483,46 @@
],
"nullable": true
},
+ "value_enc": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ },
+ {
+ "type": "string",
+ "format": "null",
+ "nullable": true
+ }
+ ],
+ "nullable": true
+ }
+ },
+ {}
+ ],
+ "nullable": true
+ },
"agent_id": {
"type": "string"
}
@@ -21561,6 +21641,20 @@
"title": "Description",
"description": "An optional description of the environment variable."
},
+ "value_enc": {
+ "anyOf": [
+ {
+ "type": "string",
+ "description": "Encrypted secret value (stored as encrypted string)",
+ "nullable": true
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Value Enc",
+ "description": "Encrypted value as Secret object"
+ },
"agent_id": {
"type": "string",
"title": "Agent Id",
@@ -30834,6 +30928,34 @@
],
"title": "Updated At",
"description": "The last update timestamp of the provider."
+ },
+ "api_key_enc": {
+ "anyOf": [
+ {
+ "type": "string",
+ "description": "Encrypted secret value (stored as encrypted string)",
+ "nullable": true
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Api Key Enc",
+ "description": "Encrypted API key as Secret object"
+ },
+ "access_key_enc": {
+ "anyOf": [
+ {
+ "type": "string",
+ "description": "Encrypted secret value (stored as encrypted string)",
+ "nullable": true
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Access Key Enc",
+ "description": "Encrypted access key as Secret object"
}
},
"additionalProperties": false,
@@ -32021,6 +32143,20 @@
"title": "Description",
"description": "An optional description of the environment variable."
},
+ "value_enc": {
+ "anyOf": [
+ {
+ "type": "string",
+ "description": "Encrypted secret value (stored as encrypted string)",
+ "nullable": true
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Value Enc",
+ "description": "Encrypted value as Secret object"
+ },
"sandbox_config_id": {
"type": "string",
"title": "Sandbox Config Id",
diff --git a/letta/schemas/environment_variables.py b/letta/schemas/environment_variables.py
index bf423e06e..0a0e53c7a 100644
--- a/letta/schemas/environment_variables.py
+++ b/letta/schemas/environment_variables.py
@@ -3,6 +3,8 @@ from typing import Optional
from pydantic import Field
from letta.schemas.letta_base import LettaBase, OrmMetadataBase
+from letta.schemas.secret import Secret
+from letta.settings import settings
# Base Environment Variable
@@ -13,6 +15,28 @@ class EnvironmentVariableBase(OrmMetadataBase):
description: Optional[str] = Field(None, description="An optional description of the environment variable.")
organization_id: Optional[str] = Field(None, description="The ID of the organization this environment variable belongs to.")
+ # Encrypted field (stored as Secret object, serialized to string for DB)
+ # Secret class handles validation and serialization automatically via __get_pydantic_core_schema__
+ value_enc: Secret | None = Field(None, description="Encrypted value as Secret object")
+
+ def get_value_secret(self) -> Secret:
+ """Get the value as a Secret object, preferring encrypted over plaintext."""
+ # If value_enc is already a Secret, return it
+ if self.value_enc is not None:
+ return self.value_enc
+ # Otherwise, create from plaintext value
+ return Secret.from_db(None, self.value)
+
+ def set_value_secret(self, secret: Secret) -> None:
+ """Set value from a Secret object, directly storing the Secret."""
+ self.value_enc = secret
+ # Also update plaintext field for dual-write during migration
+ secret_dict = secret.to_dict()
+ if not secret.was_encrypted:
+ self.value = secret_dict["plaintext"]
+ else:
+ self.value = None
+
class EnvironmentVariableCreateBase(LettaBase):
key: str = Field(..., description="The name of the environment variable.")
diff --git a/letta/schemas/providers/base.py b/letta/schemas/providers/base.py
index 8187f30a0..a88fd5a7f 100644
--- a/letta/schemas/providers/base.py
+++ b/letta/schemas/providers/base.py
@@ -8,6 +8,7 @@ from letta.schemas.enums import ProviderCategory, ProviderType
from letta.schemas.letta_base import LettaBase
from letta.schemas.llm_config import LLMConfig
from letta.schemas.llm_config_overrides import LLM_HANDLE_OVERRIDES
+from letta.schemas.secret import Secret
from letta.settings import model_settings
@@ -28,8 +29,14 @@ class Provider(ProviderBase):
organization_id: str | None = Field(None, description="The organization id of the user")
updated_at: datetime | None = Field(None, description="The last update timestamp of the provider.")
+ # Encrypted fields (stored as Secret objects, serialized to strings for DB)
+ # Secret class handles validation and serialization automatically via __get_pydantic_core_schema__
+ api_key_enc: Secret | None = Field(None, description="Encrypted API key as Secret object")
+ access_key_enc: Secret | None = Field(None, description="Encrypted access key as Secret object")
+
@model_validator(mode="after")
def default_base_url(self):
+ # Set default base URL
if self.provider_type == ProviderType.openai and self.base_url is None:
self.base_url = model_settings.openai_api_base
return self
@@ -38,6 +45,42 @@ class Provider(ProviderBase):
if not self.id:
self.id = ProviderBase.generate_id(prefix=ProviderBase.__id_prefix__)
+ def get_api_key_secret(self) -> Secret:
+ """Get the API key as a Secret object, preferring encrypted over plaintext."""
+ # If api_key_enc is already a Secret, return it
+ if self.api_key_enc is not None:
+ return self.api_key_enc
+ # Otherwise, create from plaintext api_key
+ return Secret.from_db(None, self.api_key)
+
+ def get_access_key_secret(self) -> Secret:
+ """Get the access key as a Secret object, preferring encrypted over plaintext."""
+ # If access_key_enc is already a Secret, return it
+ if self.access_key_enc is not None:
+ return self.access_key_enc
+ # Otherwise, create from plaintext access_key
+ return Secret.from_db(None, self.access_key)
+
+ def set_api_key_secret(self, secret: Secret) -> None:
+ """Set API key from a Secret object, directly storing the Secret."""
+ self.api_key_enc = secret
+ # Also update plaintext field for dual-write during migration
+ secret_dict = secret.to_dict()
+ if not secret.was_encrypted:
+ self.api_key = secret_dict["plaintext"]
+ else:
+ self.api_key = None
+
+ def set_access_key_secret(self, secret: Secret) -> None:
+ """Set access key from a Secret object, directly storing the Secret."""
+ self.access_key_enc = secret
+ # Also update plaintext field for dual-write during migration
+ secret_dict = secret.to_dict()
+ if not secret.was_encrypted:
+ self.access_key = secret_dict["plaintext"]
+ else:
+ self.access_key = None
+
async def check_api_key(self):
"""Check if the API key is valid for the provider"""
raise NotImplementedError
diff --git a/letta/schemas/secret.py b/letta/schemas/secret.py
index 45059aef7..8986a1f51 100644
--- a/letta/schemas/secret.py
+++ b/letta/schemas/secret.py
@@ -2,6 +2,7 @@ import json
from typing import Any, Dict, Optional
from pydantic import BaseModel, ConfigDict, PrivateAttr
+from pydantic_core import core_schema
from letta.helpers.crypto_utils import CryptoUtils
from letta.log import get_logger
@@ -19,16 +20,16 @@ class Secret(BaseModel):
TODO: Once we deprecate plaintext columns in the database:
- Remove the dual-write logic in to_dict()
- Remove the from_db() method's plaintext_value parameter
- - Remove the _was_encrypted flag (no longer needed for migration)
+ - Remove the was_encrypted flag (no longer needed for migration)
- Simplify get_plaintext() to only handle encrypted values
"""
- # Store the encrypted value
- _encrypted_value: Optional[str] = PrivateAttr(default=None)
- # Cache the decrypted value to avoid repeated decryption
+ # Store the encrypted value as a regular field
+ encrypted_value: Optional[str] = None
+ # Cache the decrypted value to avoid repeated decryption (not serialized for security)
_plaintext_cache: Optional[str] = PrivateAttr(default=None)
# Flag to indicate if the value was originally encrypted
- _was_encrypted: bool = PrivateAttr(default=False)
+ was_encrypted: bool = False
model_config = ConfigDict(frozen=True)
@@ -44,18 +45,16 @@ class Secret(BaseModel):
A Secret instance with the encrypted value, or plaintext if encryption unavailable
"""
if value is None:
- instance = cls()
- instance._encrypted_value = None
- instance._was_encrypted = False
- return instance
+ return cls.model_construct(encrypted_value=None, was_encrypted=False)
+
+ # Guard against double encryption - check if value is already encrypted
+ if CryptoUtils.is_encrypted(value):
+ logger.warning("Creating Secret from already-encrypted value. This can be dangerous.")
# Try to encrypt, but fall back to plaintext if no encryption key
try:
encrypted = CryptoUtils.encrypt(value)
- instance = cls()
- instance._encrypted_value = encrypted
- instance._was_encrypted = False
- return instance
+ return cls.model_construct(encrypted_value=encrypted, was_encrypted=False)
except ValueError as e:
# No encryption key available, store as plaintext
if "No encryption key configured" in str(e):
@@ -63,10 +62,8 @@ class Secret(BaseModel):
"No encryption key configured. Storing Secret value as plaintext. "
"Set LETTA_ENCRYPTION_KEY environment variable to enable encryption."
)
- instance = cls()
- instance._encrypted_value = value # Store plaintext
+ instance = cls.model_construct(encrypted_value=value, was_encrypted=False)
instance._plaintext_cache = value # Cache it
- instance._was_encrypted = False
return instance
raise # Re-raise if it's a different error
@@ -81,10 +78,7 @@ class Secret(BaseModel):
Returns:
A Secret instance
"""
- instance = cls()
- instance._encrypted_value = encrypted_value
- instance._was_encrypted = True
- return instance
+ return cls.model_construct(encrypted_value=encrypted_value, was_encrypted=True)
@classmethod
def from_db(cls, encrypted_value: Optional[str], plaintext_value: Optional[str]) -> "Secret":
@@ -114,7 +108,7 @@ class Secret(BaseModel):
Returns:
The encrypted value, or None if the secret is empty
"""
- return self._encrypted_value
+ return self.encrypted_value
def get_plaintext(self) -> Optional[str]:
"""
@@ -126,7 +120,7 @@ class Secret(BaseModel):
Returns:
The decrypted plaintext value
"""
- if self._encrypted_value is None:
+ if self.encrypted_value is None:
return None
# Use cached value if available, but only if it looks like plaintext
@@ -134,14 +128,14 @@ class Secret(BaseModel):
if self._plaintext_cache is not None:
# If we have a cache but the stored value looks encrypted and we have no key,
# we should not use the cache
- if CryptoUtils.is_encrypted(self._encrypted_value) and not CryptoUtils.is_encryption_available():
+ if CryptoUtils.is_encrypted(self.encrypted_value) and not CryptoUtils.is_encryption_available():
self._plaintext_cache = None # Clear invalid cache
else:
return self._plaintext_cache
# Decrypt and cache
try:
- plaintext = CryptoUtils.decrypt(self._encrypted_value)
+ plaintext = CryptoUtils.decrypt(self.encrypted_value)
# Cache the decrypted value (PrivateAttr fields can be mutated even with frozen=True)
self._plaintext_cache = plaintext
return plaintext
@@ -151,7 +145,7 @@ class Secret(BaseModel):
# Handle missing encryption key
if "No encryption key configured" in error_msg:
# Check if the value looks encrypted
- if CryptoUtils.is_encrypted(self._encrypted_value):
+ if CryptoUtils.is_encrypted(self.encrypted_value):
# Value was encrypted, but now we have no key - can't decrypt
logger.warning(
"Cannot decrypt Secret value - no encryption key configured. "
@@ -162,26 +156,26 @@ class Secret(BaseModel):
else:
# Value is plaintext (stored when no key was available)
logger.debug("Secret value is plaintext (stored without encryption)")
- self._plaintext_cache = self._encrypted_value
- return self._encrypted_value
+ self._plaintext_cache = self.encrypted_value
+ return self.encrypted_value
# Handle decryption failure (might be plaintext stored as such)
elif "Failed to decrypt data" in error_msg:
# Check if it might be plaintext
- if not CryptoUtils.is_encrypted(self._encrypted_value):
+ if not CryptoUtils.is_encrypted(self.encrypted_value):
# It's plaintext that was stored when no key was available
logger.debug("Secret value appears to be plaintext (stored without encryption)")
- self._plaintext_cache = self._encrypted_value
- return self._encrypted_value
+ self._plaintext_cache = self.encrypted_value
+ return self.encrypted_value
# Otherwise, it's corrupted or wrong key
logger.error("Failed to decrypt Secret value - data may be corrupted or wrong key")
raise
# Migration case: handle legacy plaintext
- elif not self._was_encrypted:
- if self._encrypted_value and not CryptoUtils.is_encrypted(self._encrypted_value):
- self._plaintext_cache = self._encrypted_value
- return self._encrypted_value
+ elif not self.was_encrypted:
+ if self.encrypted_value and not CryptoUtils.is_encrypted(self.encrypted_value):
+ self._plaintext_cache = self.encrypted_value
+ return self.encrypted_value
return None
# Re-raise for other errors
@@ -189,7 +183,7 @@ class Secret(BaseModel):
def is_empty(self) -> bool:
"""Check if the secret is empty/None."""
- return self._encrypted_value is None
+ return self.encrypted_value is None
def __str__(self) -> str:
"""String representation that doesn't expose the actual value."""
@@ -207,7 +201,7 @@ class Secret(BaseModel):
Returns both encrypted and plaintext values for dual-write during migration.
"""
- return {"encrypted": self.get_encrypted(), "plaintext": self.get_plaintext() if not self._was_encrypted else None}
+ return {"encrypted": self.get_encrypted(), "plaintext": self.get_plaintext() if not self.was_encrypted else None}
def __eq__(self, other: Any) -> bool:
"""
@@ -219,6 +213,79 @@ class Secret(BaseModel):
return False
return self.get_plaintext() == other.get_plaintext()
+ @classmethod
+ def __get_pydantic_core_schema__(cls, source_type: Any, handler) -> core_schema.CoreSchema:
+ """
+ Customize Pydantic's validation and serialization behavior for Secret fields.
+
+ This allows Secret fields to automatically:
+ - Deserialize: Convert encrypted strings from DB → Secret objects
+ - Serialize: Convert Secret objects → encrypted strings for DB
+ """
+
+ def validate_secret(value: Any) -> "Secret":
+ """Convert various input types to Secret objects."""
+ if isinstance(value, Secret):
+ return value
+ elif isinstance(value, str):
+ # String from DB is assumed to be encrypted
+ return Secret.from_encrypted(value)
+ elif isinstance(value, dict):
+ # Dict might be from Pydantic serialization - check for encrypted_value key
+ if "encrypted_value" in value:
+ # This is a serialized Secret being deserialized
+ return cls(**value)
+ elif not value or value == {}:
+ # Empty dict means None
+ return Secret.from_plaintext(None)
+ else:
+ raise ValueError(f"Cannot convert dict to Secret: {value}")
+ elif value is None:
+ return Secret.from_plaintext(None)
+ else:
+ raise ValueError(f"Cannot convert {type(value)} to Secret")
+
+ def serialize_secret(secret: "Secret") -> Optional[str]:
+ """Serialize Secret to encrypted string."""
+ if secret is None:
+ return None
+ return secret.get_encrypted()
+
+ python_schema = core_schema.chain_schema(
+ [
+ core_schema.no_info_plain_validator_function(validate_secret),
+ core_schema.is_instance_schema(cls),
+ ]
+ )
+
+ return core_schema.json_or_python_schema(
+ json_schema=python_schema,
+ python_schema=python_schema,
+ serialization=core_schema.plain_serializer_function_ser_schema(
+ serialize_secret,
+ when_used="always",
+ ),
+ )
+
+ @classmethod
+ def __get_pydantic_json_schema__(cls, core_schema: core_schema.CoreSchema, handler) -> Dict[str, Any]:
+ """
+ Define JSON schema representation for Secret fields.
+ In JSON schema (OpenAPI docs), Secret fields appear as nullable strings.
+ The actual encryption/decryption happens at runtime via __get_pydantic_core_schema__.
+ Args:
+ core_schema: The core schema for this type
+ handler: Handler for generating JSON schema
+ Returns:
+ A JSON schema dict representing this type as a nullable string
+ """
+ # Return a simple string schema for JSON schema generation
+ return {
+ "type": "string",
+ "nullable": True,
+ "description": "Encrypted secret value (stored as encrypted string)",
+ }
+
class SecretDict(BaseModel):
"""