π OpenHands: AI-Driven Development
fix: self-heal stale LiteLLM user on re-onboarding (#14932)
simonrosenberg94eba54c25ec
diff --git a/enterprise/storage/lite_llm_manager.py b/enterprise/storage/lite_llm_manager.pyindex 014953b4b0..dd2fb1e821 100644--- a/enterprise/storage/lite_llm_manager.py+++ b/enterprise/storage/lite_llm_manager.py@@ -200,6 +200,29 @@ class LiteLlmManager:)if create_user:+ # create_user is True only when no OpenHands User row exists,+ # so a pre-existing LiteLLM record under this id is a stale+ # orphan (e.g. from an account reset). Reset it so _create_user+ # rebuilds a clean record rather than re-attaching it. Best-+ # effort: a failed reset must not block onboarding β _create_user+ # below still tolerates a surviving record (409).+ if await LiteLlmManager._user_exists(client, keycloak_user_id):+ logger.info(+ 'LiteLlmManager:create_entries:reset_stale_litellm_user',+ extra={'org_id': org_id, 'user_id': keycloak_user_id},+ )+ try:+ await LiteLlmManager._delete_user(client, keycloak_user_id)+ except Exception as exc:+ logger.warning(+ 'LiteLlmManager:create_entries:reset_stale_litellm_user_failed',+ extra={+ 'org_id': org_id,+ 'user_id': keycloak_user_id,+ 'error': str(exc),+ },+ )+user_created = await LiteLlmManager._create_user(client, keycloak_user_info.get('email'), keycloak_user_id)@@ -1049,6 +1072,13 @@ class LiteLlmManager:)if not response.is_success:+ if response.status_code == 404:+ # User doesn't exist - delete is idempotent (mirrors _delete_team).+ logger.info(+ 'LiteLlmManager:_delete_user:already_deleted_or_missing',+ extra={'user_id': keycloak_user_id},+ )+ returnlogger.error('error_deleting_litellm_user',extra={diff --git a/enterprise/tests/unit/test_lite_llm_manager.py b/enterprise/tests/unit/test_lite_llm_manager.pyindex dcdaf51655..b3954505bf 100644--- a/enterprise/tests/unit/test_lite_llm_manager.py+++ b/enterprise/tests/unit/test_lite_llm_manager.py@@ -619,6 +619,267 @@ class TestLiteLlmManager:assert exc_info.value.response.status_code == 500+ @pytest.mark.asyncio+ async def test_create_entries_resets_stale_litellm_user(self, mock_settings):+ """Onboarding a brand-new user must reset a pre-existing LiteLLM record.++ A surviving LiteLLM user under the same id is a stale orphan from a+ prior teardown (e.g. an account reset), so create_entries deletes it+ before _create_user rebuilds a clean record.+ """+ mock_token_manager = MagicMock()+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(+ return_value={'email': '[email protected]'}+ )++ mock_client = AsyncMock()+ mock_client_class = MagicMock()+ mock_client_class.return_value.__aenter__.return_value = mock_client++ with (+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),+ patch('httpx.AsyncClient', mock_client_class),+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),+ patch.object(+ LiteLlmManager,+ '_team_alias_for_org',+ new=AsyncMock(return_value='Personal Workspace'),+ ),+ # Pre-check sees the orphan; post-create verify sees the clean user.+ patch.object(+ LiteLlmManager, '_user_exists', new=AsyncMock(return_value=True)+ ),+ patch.object(LiteLlmManager, '_delete_user', new=AsyncMock()) as mock_del,+ patch.object(+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)+ ) as mock_create,+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),+ patch.object(+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')+ ),+ ):+ result = await LiteLlmManager.create_entries(+ 'test-user-id', 'test-user-id', mock_settings, create_user=True+ )++ assert result is not None+ mock_del.assert_awaited_once_with(mock_client, 'test-user-id')+ mock_create.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_create_entries_skips_reset_for_fresh_litellm_user(+ self, mock_settings+ ):+ """A genuinely new user (no LiteLLM record) is created without a delete."""+ mock_token_manager = MagicMock()+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(+ return_value={'email': '[email protected]'}+ )++ mock_client = AsyncMock()+ mock_client_class = MagicMock()+ mock_client_class.return_value.__aenter__.return_value = mock_client++ with (+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),+ patch('httpx.AsyncClient', mock_client_class),+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),+ patch.object(+ LiteLlmManager,+ '_team_alias_for_org',+ new=AsyncMock(return_value='Personal Workspace'),+ ),+ # Pre-check: absent; post-create verify: present.+ patch.object(+ LiteLlmManager,+ '_user_exists',+ new=AsyncMock(side_effect=[False, True]),+ ),+ patch.object(LiteLlmManager, '_delete_user', new=AsyncMock()) as mock_del,+ patch.object(+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)+ ) as mock_create,+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),+ patch.object(+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')+ ),+ ):+ result = await LiteLlmManager.create_entries(+ 'test-user-id', 'test-user-id', mock_settings, create_user=True+ )++ assert result is not None+ mock_del.assert_not_awaited()+ mock_create.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_create_entries_reset_failure_does_not_block_onboarding(+ self, mock_settings+ ):+ """A failed stale-user reset must not block onboarding (best-effort).++ If _delete_user raises (e.g. a transient 5xx from the proxy),+ create_entries swallows it and still calls _create_user, which+ tolerates the surviving record via its 409 handling.+ """+ mock_token_manager = MagicMock()+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(+ return_value={'email': '[email protected]'}+ )++ mock_client = AsyncMock()+ mock_client_class = MagicMock()+ mock_client_class.return_value.__aenter__.return_value = mock_client++ with (+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),+ patch('httpx.AsyncClient', mock_client_class),+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),+ patch.object(+ LiteLlmManager,+ '_team_alias_for_org',+ new=AsyncMock(return_value='Personal Workspace'),+ ),+ patch.object(+ LiteLlmManager, '_user_exists', new=AsyncMock(return_value=True)+ ),+ patch.object(+ LiteLlmManager,+ '_delete_user',+ new=AsyncMock(+ side_effect=httpx.HTTPStatusError(+ '500', request=MagicMock(), response=MagicMock()+ )+ ),+ ) as mock_del,+ patch.object(+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)+ ) as mock_create,+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),+ patch.object(+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')+ ),+ ):+ result = await LiteLlmManager.create_entries(+ 'test-user-id', 'test-user-id', mock_settings, create_user=True+ )++ assert result is not None+ mock_del.assert_awaited_once()+ mock_create.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_create_entries_resets_stale_user_via_http(+ self, mock_settings, mock_response+ ):+ """Drive the real create_entries body over a mocked HTTP client.++ Unlike the method-mocked tests above, this exercises the actual+ _user_exists / _delete_user / _create_user bodies and asserts that+ POST /user/delete fires for the orphan before POST /user/new.+ """+ mock_404 = MagicMock()+ mock_404.status_code = 404+ mock_404.is_success = False+ mock_404.raise_for_status.side_effect = httpx.HTTPStatusError(+ message='Not Found', request=MagicMock(), response=mock_404+ )++ # _user_exists: orphan present on the pre-check and after recreate.+ mock_user_exists = MagicMock()+ mock_user_exists.is_success = True+ mock_user_exists.json.return_value = {'user_info': {'user_id': 'test-user-id'}}++ mock_token_manager = MagicMock()+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(+ return_value={'email': '[email protected]'}+ )++ mock_client = AsyncMock()+ # GET: _get_team (404) -> _user_exists pre-check -> _user_exists verify+ mock_client.get.side_effect = [mock_404, mock_user_exists, mock_user_exists]+ mock_client.post.return_value = mock_response++ mock_client_class = MagicMock()+ mock_client_class.return_value.__aenter__.return_value = mock_client++ with (+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),+ patch('httpx.AsyncClient', mock_client_class),+ patch.object(+ LiteLlmManager,+ '_team_alias_for_org',+ new=AsyncMock(return_value='Personal Workspace'),+ ),+ ):+ result = await LiteLlmManager.create_entries(+ 'test-user-id', 'test-user-id', mock_settings, create_user=True+ )++ assert result is not None+ post_urls = [call.args[0] for call in mock_client.post.call_args_list]+ delete_idx = next(i for i, u in enumerate(post_urls) if '/user/delete' in u)+ new_idx = next(i for i, u in enumerate(post_urls) if '/user/new' in u)+ # The orphan is deleted before the clean record is created.+ assert delete_idx < new_idx+ delete_call = mock_client.post.call_args_list[delete_idx]+ assert delete_call.kwargs['json'] == {'user_ids': ['test-user-id']}++ @pytest.mark.asyncio+ @patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com')+ @patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key')+ async def test_delete_user_idempotent_on_404(self, mock_http_client):+ """_delete_user treats 404 as success (mirrors _delete_team)."""+ response = MagicMock()+ response.is_success = False+ response.status_code = 404+ response.text = 'User not found'+ response.raise_for_status = MagicMock(+ side_effect=AssertionError('should not raise on 404')+ )+ mock_http_client.post.return_value = response++ # Must not raise.+ await LiteLlmManager._delete_user(mock_http_client, 'missing-user')+ response.raise_for_status.assert_not_called()++ @pytest.mark.asyncio+ @patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com')+ @patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key')+ async def test_delete_user_raises_on_server_error(self, mock_http_client):+ """_delete_user still surfaces non-404 failures."""+ response = MagicMock()+ response.is_success = False+ response.status_code = 500+ response.text = 'Internal server error'+ response.raise_for_status = MagicMock(+ side_effect=httpx.HTTPStatusError(+ message='500', request=MagicMock(), response=response+ )+ )+ mock_http_client.post.return_value = response++ with pytest.raises(httpx.HTTPStatusError):+ await LiteLlmManager._delete_user(mock_http_client, 'test-user-id')+@pytest.mark.asyncioasync def test_migrate_entries_missing_config(self, mock_user_settings):"""Test migrate_entries when LiteLLM config is missing."""