Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: self-heal stale LiteLLM user on re-onboarding (#14932)

simonrosenberg94eba54c25ec
diff --git a/enterprise/storage/lite_llm_manager.py b/enterprise/storage/lite_llm_manager.py
index 014953b4b0..dd2fb1e821 100644
--- a/enterprise/storage/lite_llm_manager.py
+++ b/enterprise/storage/lite_llm_manager.py
@@ -200,6 +200,29 @@ class LiteLlmManager:
)
if create_user:
+ # create_user is True only when no OpenHands User row exists,
+ # so a pre-existing LiteLLM record under this id is a stale
+ # orphan (e.g. from an account reset). Reset it so _create_user
+ # rebuilds a clean record rather than re-attaching it. Best-
+ # effort: a failed reset must not block onboarding β€” _create_user
+ # below still tolerates a surviving record (409).
+ if await LiteLlmManager._user_exists(client, keycloak_user_id):
+ logger.info(
+ 'LiteLlmManager:create_entries:reset_stale_litellm_user',
+ extra={'org_id': org_id, 'user_id': keycloak_user_id},
+ )
+ try:
+ await LiteLlmManager._delete_user(client, keycloak_user_id)
+ except Exception as exc:
+ logger.warning(
+ 'LiteLlmManager:create_entries:reset_stale_litellm_user_failed',
+ extra={
+ 'org_id': org_id,
+ 'user_id': keycloak_user_id,
+ 'error': str(exc),
+ },
+ )
+
user_created = await LiteLlmManager._create_user(
client, keycloak_user_info.get('email'), keycloak_user_id
)
@@ -1049,6 +1072,13 @@ class LiteLlmManager:
)
if not response.is_success:
+ if response.status_code == 404:
+ # User doesn't exist - delete is idempotent (mirrors _delete_team).
+ logger.info(
+ 'LiteLlmManager:_delete_user:already_deleted_or_missing',
+ extra={'user_id': keycloak_user_id},
+ )
+ return
logger.error(
'error_deleting_litellm_user',
extra={
diff --git a/enterprise/tests/unit/test_lite_llm_manager.py b/enterprise/tests/unit/test_lite_llm_manager.py
index dcdaf51655..b3954505bf 100644
--- a/enterprise/tests/unit/test_lite_llm_manager.py
+++ b/enterprise/tests/unit/test_lite_llm_manager.py
@@ -619,6 +619,267 @@ class TestLiteLlmManager:
assert exc_info.value.response.status_code == 500
+ @pytest.mark.asyncio
+ async def test_create_entries_resets_stale_litellm_user(self, mock_settings):
+ """Onboarding a brand-new user must reset a pre-existing LiteLLM record.
+
+ A surviving LiteLLM user under the same id is a stale orphan from a
+ prior teardown (e.g. an account reset), so create_entries deletes it
+ before _create_user rebuilds a clean record.
+ """
+ mock_token_manager = MagicMock()
+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(
+ return_value={'email': '[email protected]'}
+ )
+
+ mock_client = AsyncMock()
+ mock_client_class = MagicMock()
+ mock_client_class.return_value.__aenter__.return_value = mock_client
+
+ with (
+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),
+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),
+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),
+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),
+ patch('httpx.AsyncClient', mock_client_class),
+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),
+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager,
+ '_team_alias_for_org',
+ new=AsyncMock(return_value='Personal Workspace'),
+ ),
+ # Pre-check sees the orphan; post-create verify sees the clean user.
+ patch.object(
+ LiteLlmManager, '_user_exists', new=AsyncMock(return_value=True)
+ ),
+ patch.object(LiteLlmManager, '_delete_user', new=AsyncMock()) as mock_del,
+ patch.object(
+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)
+ ) as mock_create,
+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),
+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')
+ ),
+ ):
+ result = await LiteLlmManager.create_entries(
+ 'test-user-id', 'test-user-id', mock_settings, create_user=True
+ )
+
+ assert result is not None
+ mock_del.assert_awaited_once_with(mock_client, 'test-user-id')
+ mock_create.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_create_entries_skips_reset_for_fresh_litellm_user(
+ self, mock_settings
+ ):
+ """A genuinely new user (no LiteLLM record) is created without a delete."""
+ mock_token_manager = MagicMock()
+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(
+ return_value={'email': '[email protected]'}
+ )
+
+ mock_client = AsyncMock()
+ mock_client_class = MagicMock()
+ mock_client_class.return_value.__aenter__.return_value = mock_client
+
+ with (
+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),
+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),
+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),
+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),
+ patch('httpx.AsyncClient', mock_client_class),
+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),
+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager,
+ '_team_alias_for_org',
+ new=AsyncMock(return_value='Personal Workspace'),
+ ),
+ # Pre-check: absent; post-create verify: present.
+ patch.object(
+ LiteLlmManager,
+ '_user_exists',
+ new=AsyncMock(side_effect=[False, True]),
+ ),
+ patch.object(LiteLlmManager, '_delete_user', new=AsyncMock()) as mock_del,
+ patch.object(
+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)
+ ) as mock_create,
+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),
+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')
+ ),
+ ):
+ result = await LiteLlmManager.create_entries(
+ 'test-user-id', 'test-user-id', mock_settings, create_user=True
+ )
+
+ assert result is not None
+ mock_del.assert_not_awaited()
+ mock_create.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_create_entries_reset_failure_does_not_block_onboarding(
+ self, mock_settings
+ ):
+ """A failed stale-user reset must not block onboarding (best-effort).
+
+ If _delete_user raises (e.g. a transient 5xx from the proxy),
+ create_entries swallows it and still calls _create_user, which
+ tolerates the surviving record via its 409 handling.
+ """
+ mock_token_manager = MagicMock()
+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(
+ return_value={'email': '[email protected]'}
+ )
+
+ mock_client = AsyncMock()
+ mock_client_class = MagicMock()
+ mock_client_class.return_value.__aenter__.return_value = mock_client
+
+ with (
+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),
+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),
+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),
+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),
+ patch('httpx.AsyncClient', mock_client_class),
+ patch.object(LiteLlmManager, '_get_team', new=AsyncMock(return_value=None)),
+ patch.object(LiteLlmManager, '_create_team', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager,
+ '_team_alias_for_org',
+ new=AsyncMock(return_value='Personal Workspace'),
+ ),
+ patch.object(
+ LiteLlmManager, '_user_exists', new=AsyncMock(return_value=True)
+ ),
+ patch.object(
+ LiteLlmManager,
+ '_delete_user',
+ new=AsyncMock(
+ side_effect=httpx.HTTPStatusError(
+ '500', request=MagicMock(), response=MagicMock()
+ )
+ ),
+ ) as mock_del,
+ patch.object(
+ LiteLlmManager, '_create_user', new=AsyncMock(return_value=True)
+ ) as mock_create,
+ patch.object(LiteLlmManager, '_add_user_to_team', new=AsyncMock()),
+ patch.object(LiteLlmManager, '_delete_key_by_alias', new=AsyncMock()),
+ patch.object(
+ LiteLlmManager, '_generate_key', new=AsyncMock(return_value='new-key')
+ ),
+ ):
+ result = await LiteLlmManager.create_entries(
+ 'test-user-id', 'test-user-id', mock_settings, create_user=True
+ )
+
+ assert result is not None
+ mock_del.assert_awaited_once()
+ mock_create.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_create_entries_resets_stale_user_via_http(
+ self, mock_settings, mock_response
+ ):
+ """Drive the real create_entries body over a mocked HTTP client.
+
+ Unlike the method-mocked tests above, this exercises the actual
+ _user_exists / _delete_user / _create_user bodies and asserts that
+ POST /user/delete fires for the orphan before POST /user/new.
+ """
+ mock_404 = MagicMock()
+ mock_404.status_code = 404
+ mock_404.is_success = False
+ mock_404.raise_for_status.side_effect = httpx.HTTPStatusError(
+ message='Not Found', request=MagicMock(), response=mock_404
+ )
+
+ # _user_exists: orphan present on the pre-check and after recreate.
+ mock_user_exists = MagicMock()
+ mock_user_exists.is_success = True
+ mock_user_exists.json.return_value = {'user_info': {'user_id': 'test-user-id'}}
+
+ mock_token_manager = MagicMock()
+ mock_token_manager.return_value.get_user_info_from_user_id = AsyncMock(
+ return_value={'email': '[email protected]'}
+ )
+
+ mock_client = AsyncMock()
+ # GET: _get_team (404) -> _user_exists pre-check -> _user_exists verify
+ mock_client.get.side_effect = [mock_404, mock_user_exists, mock_user_exists]
+ mock_client.post.return_value = mock_response
+
+ mock_client_class = MagicMock()
+ mock_client_class.return_value.__aenter__.return_value = mock_client
+
+ with (
+ patch.dict(os.environ, {'LOCAL_DEPLOYMENT': ''}),
+ patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key'),
+ patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com'),
+ patch('storage.lite_llm_manager.TokenManager', mock_token_manager),
+ patch('httpx.AsyncClient', mock_client_class),
+ patch.object(
+ LiteLlmManager,
+ '_team_alias_for_org',
+ new=AsyncMock(return_value='Personal Workspace'),
+ ),
+ ):
+ result = await LiteLlmManager.create_entries(
+ 'test-user-id', 'test-user-id', mock_settings, create_user=True
+ )
+
+ assert result is not None
+ post_urls = [call.args[0] for call in mock_client.post.call_args_list]
+ delete_idx = next(i for i, u in enumerate(post_urls) if '/user/delete' in u)
+ new_idx = next(i for i, u in enumerate(post_urls) if '/user/new' in u)
+ # The orphan is deleted before the clean record is created.
+ assert delete_idx < new_idx
+ delete_call = mock_client.post.call_args_list[delete_idx]
+ assert delete_call.kwargs['json'] == {'user_ids': ['test-user-id']}
+
+ @pytest.mark.asyncio
+ @patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com')
+ @patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key')
+ async def test_delete_user_idempotent_on_404(self, mock_http_client):
+ """_delete_user treats 404 as success (mirrors _delete_team)."""
+ response = MagicMock()
+ response.is_success = False
+ response.status_code = 404
+ response.text = 'User not found'
+ response.raise_for_status = MagicMock(
+ side_effect=AssertionError('should not raise on 404')
+ )
+ mock_http_client.post.return_value = response
+
+ # Must not raise.
+ await LiteLlmManager._delete_user(mock_http_client, 'missing-user')
+ response.raise_for_status.assert_not_called()
+
+ @pytest.mark.asyncio
+ @patch('storage.lite_llm_manager.LITE_LLM_API_URL', 'http://test.com')
+ @patch('storage.lite_llm_manager.LITE_LLM_API_KEY', 'test-key')
+ async def test_delete_user_raises_on_server_error(self, mock_http_client):
+ """_delete_user still surfaces non-404 failures."""
+ response = MagicMock()
+ response.is_success = False
+ response.status_code = 500
+ response.text = 'Internal server error'
+ response.raise_for_status = MagicMock(
+ side_effect=httpx.HTTPStatusError(
+ message='500', request=MagicMock(), response=response
+ )
+ )
+ mock_http_client.post.return_value = response
+
+ with pytest.raises(httpx.HTTPStatusError):
+ await LiteLlmManager._delete_user(mock_http_client, 'test-user-id')
+
@pytest.mark.asyncio
async def test_migrate_entries_missing_config(self, mock_user_settings):
"""Test migrate_entries when LiteLLM config is missing."""