π OpenHands: AI-Driven Development
fix(backend): preserve users custom llm settings during settings migrations (#12134)
Hiep Le58a70e8b0d36
diff --git a/enterprise/storage/saas_settings_store.py b/enterprise/storage/saas_settings_store.pyindex fd64924263..30242daad0 100644--- a/enterprise/storage/saas_settings_store.py+++ b/enterprise/storage/saas_settings_store.py@@ -19,6 +19,7 @@ from server.constants import (LITE_LLM_API_URL,LITE_LLM_TEAM_ID,REQUIRE_PAYMENT,+ USER_SETTINGS_VERSION_TO_MODEL,get_default_litellm_model,)from server.logger import logger@@ -202,6 +203,53 @@ class SaasSettingsStore(SettingsStore):)return None+ def _has_custom_settings(+ self, settings: Settings, old_user_version: int | None+ ) -> bool:+ """+ Check if user has custom LLM settings that should be preserved.+ Returns True if user customized either model or base_url.++ Args:+ settings: The user's current settings+ old_user_version: The user's old settings version, if any++ Returns:+ True if user has custom settings, False if using old defaults+ """+ # Normalize values+ user_model = (+ settings.llm_model.strip()+ if settings.llm_model and settings.llm_model.strip()+ else None+ )+ user_base_url = (+ settings.llm_base_url.strip()+ if settings.llm_base_url and settings.llm_base_url.strip()+ else None+ )++ # Custom base_url = definitely custom settings (BYOK)+ if user_base_url and user_base_url != LITE_LLM_API_URL:+ return True++ # No model set = using defaults+ if not user_model:+ return False++ # Check if model matches old version's default+ if (+ old_user_version+ and old_user_version < CURRENT_USER_SETTINGS_VERSION+ and old_user_version in USER_SETTINGS_VERSION_TO_MODEL+ ):+ old_default_base = USER_SETTINGS_VERSION_TO_MODEL[old_user_version]+ user_model_base = user_model.split('/')[-1]+ if user_model_base == old_default_base:+ return False # Matches old default++ return True # Custom model+async def update_settings_with_litellm_default(self, settings: Settings) -> Settings | None:@@ -213,6 +261,17 @@ class SaasSettingsStore(SettingsStore):return Nonelocal_deploy = os.environ.get('LOCAL_DEPLOYMENT', None)key = LITE_LLM_API_KEY++ # Check if user has custom settings+ has_custom = self._has_custom_settings(settings, settings.user_version)++ # Determine model to use (needed before LiteLLM user creation)+ llm_model_to_use = (+ settings.llm_model+ if has_custom and settings.llm_model+ else get_default_litellm_model()+ )+if not local_deploy:# Get user info to add to litellmtoken_manager = TokenManager()@@ -276,7 +335,7 @@ class SaasSettingsStore(SettingsStore):# Create the new litellm userresponse = await self._create_user_in_lite_llm(- client, email, max_budget, spend+ client, email, max_budget, spend, llm_model_to_use)if not response.is_success:logger.warning(@@ -285,7 +344,7 @@ class SaasSettingsStore(SettingsStore):)# Litellm insists on unique email addresses - it is possible the email address was registered with a different user.response = await self._create_user_in_lite_llm(- client, None, max_budget, spend+ client, None, max_budget, spend, llm_model_to_use)# User failed to create in litellm - this is an unforseen error state...@@ -311,11 +370,17 @@ class SaasSettingsStore(SettingsStore):extra={'user_id': self.user_id},)+ if has_custom:+ settings.llm_model = settings.llm_model or get_default_litellm_model()+ settings.llm_base_url = settings.llm_base_url or LITE_LLM_API_URL+ settings.llm_api_key = settings.llm_api_key or SecretStr(key)+ else:+ settings.llm_model = get_default_litellm_model()+ settings.llm_base_url = LITE_LLM_API_URL+ settings.llm_api_key = SecretStr(key)+settings.agent = 'CodeActAgent'- # Use the model corresponding to the current user settings version- settings.llm_model = get_default_litellm_model()- settings.llm_api_key = SecretStr(key)- settings.llm_base_url = LITE_LLM_API_URL+return settings@classmethod@@ -398,7 +463,12 @@ class SaasSettingsStore(SettingsStore):)async def _create_user_in_lite_llm(- self, client: httpx.AsyncClient, email: str | None, max_budget: int, spend: int+ self,+ client: httpx.AsyncClient,+ email: str | None,+ max_budget: int,+ spend: int,+ llm_model: str,):response = await client.post(f'{LITE_LLM_API_URL}/user/new',@@ -413,7 +483,7 @@ class SaasSettingsStore(SettingsStore):'send_invite_email': False,'metadata': {'version': CURRENT_USER_SETTINGS_VERSION,- 'model': get_default_litellm_model(),+ 'model': llm_model,},'key_alias': f'OpenHands Cloud - user {self.user_id}',},diff --git a/enterprise/tests/unit/test_saas_settings_store.py b/enterprise/tests/unit/test_saas_settings_store.pyindex 50d7ab0b12..56f6ad3e2f 100644--- a/enterprise/tests/unit/test_saas_settings_store.py+++ b/enterprise/tests/unit/test_saas_settings_store.py@@ -6,6 +6,7 @@ from server.constants import (CURRENT_USER_SETTINGS_VERSION,LITE_LLM_API_URL,LITE_LLM_TEAM_ID,+ get_default_litellm_model,)from storage.saas_settings_store import SaasSettingsStorefrom storage.user_settings import UserSettings@@ -393,10 +394,11 @@ async def test_create_user_in_lite_llm(settings_store):mock_response = AsyncMock()mock_response.is_success = Truemock_client.post.return_value = mock_response+ test_model = 'custom-model/test-model'# Test with emailawait settings_store._create_user_in_lite_llm(- mock_client, '[email protected]', 50, 10+ mock_client, '[email protected]', 50, 10, test_model)# Get the actual call arguments@@ -412,11 +414,11 @@ async def test_create_user_in_lite_llm(settings_store):assert call_args['json']['auto_create_key'] is Trueassert call_args['json']['send_invite_email'] is Falseassert call_args['json']['metadata']['version'] == CURRENT_USER_SETTINGS_VERSION- assert 'model' in call_args['json']['metadata']+ assert call_args['json']['metadata']['model'] == test_model# Test with None emailmock_client.post.reset_mock()- await settings_store._create_user_in_lite_llm(mock_client, None, 25, 15)+ await settings_store._create_user_in_lite_llm(mock_client, None, 25, 15, test_model)# Get the actual call argumentscall_args = mock_client.post.call_args[1]@@ -431,12 +433,12 @@ async def test_create_user_in_lite_llm(settings_store):assert call_args['json']['auto_create_key'] is Trueassert call_args['json']['send_invite_email'] is Falseassert call_args['json']['metadata']['version'] == CURRENT_USER_SETTINGS_VERSION- assert 'model' in call_args['json']['metadata']+ assert call_args['json']['metadata']['model'] == test_model# Verify response is returned correctlyassert (await settings_store._create_user_in_lite_llm(- mock_client, '[email protected]', 30, 7+ mock_client, '[email protected]', 30, 7, test_model)== mock_response)@@ -464,3 +466,808 @@ async def test_encryption(settings_store):# But we should be able to decrypt it when loadingloaded_settings = await settings_store.load()assert loaded_settings.llm_api_key.get_secret_value() == 'secret_key'+++async def test_update_settings_with_litellm_default_preserves_custom_model(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has a custom LLM model set+ custom_model = 'anthropic/claude-3-5-sonnet-20241022'+ settings = Settings(llm_model=custom_model)++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Custom model is preserved+ assert updated_settings is not None+ assert updated_settings.llm_model == custom_model+ assert updated_settings.agent == 'CodeActAgent'+ assert updated_settings.llm_api_key is not None++ # Assert: LiteLLM metadata contains user's custom model+ call_args = mock_litellm_api.return_value.__aenter__.return_value.post.call_args[1]+ assert call_args['json']['metadata']['model'] == custom_model+++async def test_update_settings_with_litellm_default_uses_default_when_no_model(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has no model set (new user scenario)+ settings = Settings()++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default model is assigned+ assert updated_settings is not None+ expected_default = get_default_litellm_model()+ assert updated_settings.llm_model == expected_default+ assert updated_settings.agent == 'CodeActAgent'++ # Assert: LiteLLM metadata contains default model+ call_args = mock_litellm_api.return_value.__aenter__.return_value.post.call_args[1]+ assert call_args['json']['metadata']['model'] == expected_default+++async def test_update_settings_with_litellm_default_handles_empty_string_model(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has empty string as model (edge case)+ settings = Settings(llm_model='')++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default model is used (empty string treated as no model)+ assert updated_settings is not None+ expected_default = get_default_litellm_model()+ assert updated_settings.llm_model == expected_default+++async def test_update_settings_with_litellm_default_handles_whitespace_model(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has whitespace-only model (edge case)+ settings = Settings(llm_model=' ')++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default model is used (whitespace treated as no model)+ assert updated_settings is not None+ expected_default = get_default_litellm_model()+ assert updated_settings.llm_model == expected_default+++async def test_update_settings_with_litellm_default_preserves_custom_api_key(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has a custom API key and custom model (so has_custom=True)+ custom_api_key = 'sk-custom-user-api-key-12345'+ custom_model = 'gpt-4'+ settings = Settings(llm_model=custom_model, llm_api_key=SecretStr(custom_api_key))++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Custom API key is preserved when user has custom settings+ assert updated_settings is not None+ assert updated_settings.llm_api_key.get_secret_value() == custom_api_key+ assert updated_settings.llm_api_key.get_secret_value() != 'test_api_key'+++async def test_update_settings_with_litellm_default_preserves_custom_base_url(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has a custom base URL+ custom_base_url = 'https://api.custom-llm-provider.com/v1'+ settings = Settings(llm_base_url=custom_base_url)++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Custom base URL is preserved+ assert updated_settings is not None+ assert updated_settings.llm_base_url == custom_base_url+ assert updated_settings.llm_base_url != LITE_LLM_API_URL+++async def test_update_settings_with_litellm_default_preserves_custom_api_key_and_base_url(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has both custom API key and base URL+ custom_api_key = 'sk-custom-user-api-key-67890'+ custom_base_url = 'https://api.another-llm-provider.com/v1'+ custom_model = 'openai/gpt-4'+ settings = Settings(+ llm_model=custom_model,+ llm_api_key=SecretStr(custom_api_key),+ llm_base_url=custom_base_url,+ )++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: All custom settings are preserved+ assert updated_settings is not None+ assert updated_settings.llm_model == custom_model+ assert updated_settings.llm_api_key.get_secret_value() == custom_api_key+ assert updated_settings.llm_base_url == custom_base_url+++async def test_update_settings_with_litellm_default_uses_default_api_key_when_none(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has no API key set+ settings = Settings(llm_api_key=None)++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default LiteLLM API key is assigned+ assert updated_settings is not None+ assert updated_settings.llm_api_key is not None+ assert updated_settings.llm_api_key.get_secret_value() == 'test_api_key'+++async def test_update_settings_with_litellm_default_uses_default_base_url_when_none(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has no base URL set+ settings = Settings(llm_base_url=None)++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://test.url'),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default LiteLLM base URL is assigned (using mocked value)+ assert updated_settings is not None+ assert updated_settings.llm_base_url == 'http://test.url'+++async def test_update_settings_with_litellm_default_handles_empty_api_key(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has empty string as API key (edge case)+ settings = Settings(llm_api_key=SecretStr(''))++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default API key is used (empty string treated as no key)+ assert updated_settings is not None+ assert updated_settings.llm_api_key.get_secret_value() == 'test_api_key'+++async def test_update_settings_with_litellm_default_handles_empty_base_url(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has empty string as base URL (edge case)+ settings = Settings(llm_base_url='')++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://test.url'),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default base URL is used (empty string treated as no URL)+ assert updated_settings is not None+ assert updated_settings.llm_base_url == 'http://test.url'+++async def test_update_settings_with_litellm_default_handles_whitespace_api_key(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has whitespace-only API key (edge case)+ settings = Settings(llm_api_key=SecretStr(' '))++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default API key is used (whitespace treated as no key)+ assert updated_settings is not None+ assert updated_settings.llm_api_key.get_secret_value() == 'test_api_key'+++async def test_update_settings_with_litellm_default_handles_whitespace_base_url(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User has whitespace-only base URL (edge case)+ settings = Settings(llm_base_url=' ')++ with (+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://test.url'),+ ):+ # Act: Update settings with LiteLLM defaults+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Default base URL is used (whitespace treated as no URL)+ assert updated_settings is not None+ assert updated_settings.llm_base_url == 'http://test.url'+++# Tests for version migration and helper methods+++async def test_has_custom_settings_with_custom_base_url(settings_store):+ # Arrange: User with custom base URL (BYOR)+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_base_url='http://custom.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: Custom base URL detected+ assert has_custom is True+++async def test_has_custom_settings_with_default_base_url(settings_store):+ # Arrange: User with default base URL+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: No custom settings (no model set)+ assert has_custom is False+++async def test_has_custom_settings_with_no_model(settings_store):+ # Arrange: User with no model set+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_model=None, llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: No custom settings (using defaults)+ assert has_custom is False+++async def test_has_custom_settings_with_empty_model(settings_store):+ # Arrange: User with empty model+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_model='', llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: No custom settings (empty treated as no model)+ assert has_custom is False+++async def test_has_custom_settings_with_whitespace_model(settings_store):+ # Arrange: User with whitespace-only model+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_model=' ', llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: No custom settings (whitespace treated as no model)+ assert has_custom is False+++async def test_has_custom_settings_with_custom_model(settings_store):+ # Arrange: User with custom model+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(llm_model='gpt-4', llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: Custom model detected+ assert has_custom is True+++async def test_has_custom_settings_matches_old_default_model(settings_store):+ # Arrange: User with old version and model matching old default+ with (+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022'},+ ),+ ):+ settings = Settings(+ llm_model='litellm_proxy/prod/claude-3-5-sonnet-20241022',+ llm_base_url='http://default.url',+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, 1)++ # Assert: Matches old default, so not custom+ assert has_custom is False+++async def test_has_custom_settings_matches_old_default_by_base_name(settings_store):+ # Arrange: User with old version and model matching old default by base name+ with (+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022'},+ ),+ ):+ settings = Settings(+ llm_model='anthropic/claude-3-5-sonnet-20241022',+ llm_base_url='http://default.url',+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, 1)++ # Assert: Matches old default by base name, so not custom+ assert has_custom is False+++async def test_has_custom_settings_with_old_version_but_custom_model(settings_store):+ # Arrange: User with old version but custom model+ with (+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022'},+ ),+ ):+ settings = Settings(llm_model='gpt-4', llm_base_url='http://default.url')++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, 1)++ # Assert: Custom model detected+ assert has_custom is True+++async def test_has_custom_settings_with_current_version(settings_store):+ # Arrange: User with current version+ with (+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022', 5: 'claude-opus-4-5-20251101'},+ ),+ ):+ settings = Settings(+ llm_model='claude-3-5-sonnet-20241022', llm_base_url='http://default.url'+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, 5)++ # Assert: Current version, so model is custom (not old default)+ assert has_custom is True+++async def test_has_custom_settings_with_none_version(settings_store):+ # Arrange: User with no version+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(+ llm_model='claude-3-5-sonnet-20241022', llm_base_url='http://default.url'+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: No version, so model is custom+ assert has_custom is True+++async def test_has_custom_settings_with_invalid_version(settings_store):+ # Arrange: User with invalid version+ with (+ patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022'},+ ),+ ):+ settings = Settings(+ llm_model='claude-3-5-sonnet-20241022', llm_base_url='http://default.url'+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, 99)++ # Assert: Invalid version, so model is custom+ assert has_custom is True+++async def test_has_custom_settings_normalizes_whitespace(settings_store):+ # Arrange: Settings with whitespace in values+ with patch('storage.saas_settings_store.LITE_LLM_API_URL', 'http://default.url'):+ settings = Settings(+ llm_model=' claude-3-5-sonnet-20241022 ',+ llm_base_url=' http://default.url ',+ )++ # Act: Check if has custom settings+ has_custom = settings_store._has_custom_settings(settings, None)++ # Assert: Whitespace is normalized, custom model detected+ assert has_custom is True+++async def test_update_settings_upgrades_user_from_old_defaults(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User with old version using old defaults+ old_version = 1+ old_model = 'litellm_proxy/prod/claude-3-5-sonnet-20241022'+ settings = Settings(llm_model=old_model, llm_base_url=LITE_LLM_API_URL)++ # Use a consistent test URL+ test_base_url = 'http://test.url'++ with (+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022', 5: 'claude-opus-4-5-20251101'},+ ),+ patch(+ 'storage.saas_settings_store.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022', 5: 'claude-opus-4-5-20251101'},+ ),+ patch('server.constants.CURRENT_USER_SETTINGS_VERSION', 5),+ patch('storage.saas_settings_store.CURRENT_USER_SETTINGS_VERSION', 5),+ patch('storage.saas_settings_store.LITE_LLM_API_URL', test_base_url),+ patch(+ 'storage.saas_settings_store.get_default_litellm_model',+ return_value='litellm_proxy/prod/claude-opus-4-5-20251101',+ ),+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ ):+ # Create existing user settings with old version+ with session_maker() as session:+ existing_settings = UserSettings(+ keycloak_user_id=settings_store.user_id,+ user_version=old_version,+ llm_model=old_model,+ llm_base_url=test_base_url,+ )+ session.add(existing_settings)+ session.commit()++ # Update settings to use test_base_url+ # Set user_version to match the database so _has_custom_settings can detect old defaults+ settings = Settings(+ llm_model=old_model, llm_base_url=test_base_url, user_version=old_version+ )++ # Act: Update settings+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Settings upgraded to new defaults+ assert updated_settings is not None+ assert (+ updated_settings.llm_model == 'litellm_proxy/prod/claude-opus-4-5-20251101'+ )+ assert updated_settings.llm_base_url == test_base_url+++async def test_update_settings_preserves_custom_settings_during_upgrade(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User with old version but custom settings+ old_version = 1+ custom_model = 'gpt-4'+ custom_base_url = 'http://custom.url'+ settings = Settings(llm_model=custom_model, llm_base_url=custom_base_url)++ with (+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch(+ 'server.constants.USER_SETTINGS_VERSION_TO_MODEL',+ {1: 'claude-3-5-sonnet-20241022'},+ ),+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ ):+ # Create existing user settings with old version+ with session_maker() as session:+ existing_settings = UserSettings(+ keycloak_user_id=settings_store.user_id,+ user_version=old_version,+ llm_model=custom_model,+ llm_base_url=custom_base_url,+ )+ session.add(existing_settings)+ session.commit()++ # Act: Update settings+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Custom settings preserved+ assert updated_settings is not None+ assert updated_settings.llm_model == custom_model+ assert updated_settings.llm_base_url == custom_base_url+++async def test_update_settings_migrates_billing_margin_v3_to_v4(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User with version 3 and billing margin+ old_version = 3+ billing_margin = 2.0+ max_budget = 10.0+ spend = 5.0++ settings = Settings()++ mock_get_response = AsyncMock()+ mock_get_response.is_success = True+ mock_get_response.json = MagicMock(+ return_value={'user_info': {'max_budget': max_budget, 'spend': spend}}+ )++ mock_post_response = AsyncMock()+ mock_post_response.is_success = True+ mock_post_response.json = MagicMock(return_value={'key': 'test_api_key'})++ with (+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('httpx.AsyncClient') as mock_client,+ ):+ mock_client.return_value.__aenter__.return_value.get.return_value = (+ mock_get_response+ )+ mock_client.return_value.__aenter__.return_value.post.return_value = (+ mock_post_response+ )++ # Create existing user settings with version 3 and billing margin+ with session_maker() as session:+ existing_settings = UserSettings(+ keycloak_user_id=settings_store.user_id,+ user_version=old_version,+ billing_margin=billing_margin,+ )+ session.add(existing_settings)+ session.commit()++ # Act: Update settings+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Settings updated+ assert updated_settings is not None++ # Assert: Billing margin applied to budget+ call_args = mock_client.return_value.__aenter__.return_value.post.call_args[1]+ assert call_args['json']['max_budget'] == max_budget * billing_margin+ assert call_args['json']['spend'] == spend * billing_margin++ # Assert: Billing margin reset to 1.0+ with session_maker() as session:+ updated_user_settings = (+ session.query(UserSettings)+ .filter(UserSettings.keycloak_user_id == settings_store.user_id)+ .first()+ )+ assert updated_user_settings.billing_margin == 1.0+++async def test_update_settings_skips_billing_margin_migration_when_already_v4(+ settings_store, mock_litellm_api, session_maker+):+ # Arrange: User with version 4+ version = 4+ billing_margin = 2.0+ max_budget = 10.0+ spend = 5.0++ settings = Settings()++ mock_get_response = AsyncMock()+ mock_get_response.is_success = True+ mock_get_response.json = MagicMock(+ return_value={'user_info': {'max_budget': max_budget, 'spend': spend}}+ )++ mock_post_response = AsyncMock()+ mock_post_response.is_success = True+ mock_post_response.json = MagicMock(return_value={'key': 'test_api_key'})++ with (+ patch('storage.saas_settings_store.session_maker', session_maker),+ patch(+ 'server.auth.token_manager.TokenManager.get_user_info_from_user_id',+ AsyncMock(return_value={'email': '[email protected]'}),+ ),+ patch('httpx.AsyncClient') as mock_client,+ ):+ mock_client.return_value.__aenter__.return_value.get.return_value = (+ mock_get_response+ )+ mock_client.return_value.__aenter__.return_value.post.return_value = (+ mock_post_response+ )++ # Create existing user settings with version 4+ with session_maker() as session:+ existing_settings = UserSettings(+ keycloak_user_id=settings_store.user_id,+ user_version=version,+ billing_margin=billing_margin,+ )+ session.add(existing_settings)+ session.commit()++ # Act: Update settings+ updated_settings = await settings_store.update_settings_with_litellm_default(+ settings+ )++ # Assert: Settings updated+ assert updated_settings is not None++ # Assert: Billing margin NOT applied (version >= 4)+ call_args = mock_client.return_value.__aenter__.return_value.post.call_args[1]+ assert call_args['json']['max_budget'] == max_budget+ assert call_args['json']['spend'] == spenddiff --git a/openhands/storage/data_models/settings.py b/openhands/storage/data_models/settings.pyindex 0dc9b99e62..1f9fbe7bdd 100644--- a/openhands/storage/data_models/settings.py+++ b/openhands/storage/data_models/settings.py@@ -30,6 +30,7 @@ class Settings(BaseModel):llm_model: str | None = Nonellm_api_key: SecretStr | None = Nonellm_base_url: str | None = None+ user_version: int | None = Noneremote_runtime_resource_factor: int | None = None# Planned to be removed from settingssecrets_store: Secrets = Field(default_factory=Secrets, frozen=True)