π OpenHands: AI-Driven Development
Improve budget exceeded error handling in V1 callback processors (#13219)
Rohit Malhotrac32934ed2fc3
diff --git a/enterprise/integrations/github/github_v1_callback_processor.py b/enterprise/integrations/github/github_v1_callback_processor.pyindex 7dc82da54e..541cb27377 100644--- a/enterprise/integrations/github/github_v1_callback_processor.py+++ b/enterprise/integrations/github/github_v1_callback_processor.py@@ -4,7 +4,8 @@ from uuid import UUIDimport httpxfrom github import Auth, Github, GithubIntegration-from integrations.utils import CONVERSATION_URL, get_summary_instruction+from integrations.utils import get_summary_instruction+from integrations.v1_utils import handle_callback_errorfrom pydantic import Fieldfrom server.auth.constants import GITHUB_APP_CLIENT_ID, GITHUB_APP_PRIVATE_KEY@@ -42,7 +43,6 @@ class GithubV1CallbackProcessor(EventCallbackProcessor):event: Event,) -> EventCallbackResult | None:"""Process events for GitHub V1 integration."""-# Only handle ConversationStateUpdateEventif not isinstance(event, ConversationStateUpdateEvent):return None@@ -78,25 +78,20 @@ class GithubV1CallbackProcessor(EventCallbackProcessor):detail=summary,)except Exception as e:- _logger.exception('[GitHub V1] Error processing callback: %s', e)-- # Only try to post error to GitHub if we have basic requirements- try:- # Check if we have installation ID and credentials before posting- if (- self.github_view_data.get('installation_id')- and GITHUB_APP_CLIENT_ID- and GITHUB_APP_PRIVATE_KEY- ):- await self._post_summary_to_github(- f'OpenHands encountered an error: **{str(e)}**.\n\n'- f'[See the conversation]({CONVERSATION_URL.format(conversation_id)})'- 'for more information.'- )- except Exception as post_error:- _logger.warning(- '[GitHub V1] Failed to post error message to GitHub: %s', post_error- )+ # Check if we have installation ID and credentials before posting+ can_post_error = bool(+ self.github_view_data.get('installation_id')+ and GITHUB_APP_CLIENT_ID+ and GITHUB_APP_PRIVATE_KEY+ )+ await handle_callback_error(+ error=e,+ conversation_id=conversation_id,+ service_name='GitHub',+ service_logger=_logger,+ can_post_error=can_post_error,+ post_error_func=self._post_summary_to_github,+ )return EventCallbackResult(status=EventCallbackResultStatus.ERROR,@@ -167,8 +162,8 @@ class GithubV1CallbackProcessor(EventCallbackProcessor):send_message_request = AskAgentRequest(question=message_content)url = (- f'{agent_server_url.rstrip("/")}'- f'/api/conversations/{conversation_id}/ask_agent'+ f"{agent_server_url.rstrip('/')}"+ f"/api/conversations/{conversation_id}/ask_agent")headers = {'X-Session-API-Key': session_api_key}payload = send_message_request.model_dump()@@ -230,8 +225,7 @@ class GithubV1CallbackProcessor(EventCallbackProcessor):# -------------------------------------------------------------------------async def _request_summary(self, conversation_id: UUID) -> str:- """- Ask the agent to produce a summary of its work and return the agent response.+ """Ask the agent to produce a summary of its work and return the agent response.NOTE: This method now returns a string (the agent server's response text)and raises exceptions on errors. The wrapping into EventCallbackResultdiff --git a/enterprise/integrations/gitlab/gitlab_v1_callback_processor.py b/enterprise/integrations/gitlab/gitlab_v1_callback_processor.pyindex 8fb6521d98..2fe4d6fc72 100644--- a/enterprise/integrations/gitlab/gitlab_v1_callback_processor.py+++ b/enterprise/integrations/gitlab/gitlab_v1_callback_processor.py@@ -3,7 +3,8 @@ from typing import Anyfrom uuid import UUIDimport httpx-from integrations.utils import CONVERSATION_URL, get_summary_instruction+from integrations.utils import get_summary_instruction+from integrations.v1_utils import handle_callback_errorfrom pydantic import Fieldfrom openhands.agent_server.models import AskAgentRequest, AskAgentResponse@@ -75,20 +76,15 @@ class GitlabV1CallbackProcessor(EventCallbackProcessor):detail=summary,)except Exception as e:- _logger.exception('[GitLab V1] Error processing callback: %s', e)-- # Only try to post error to GitLab if we have basic requirements- try:- if self.gitlab_view_data.get('keycloak_user_id'):- await self._post_summary_to_gitlab(- f'OpenHands encountered an error: **{str(e)}**.\n\n'- f'[See the conversation]({CONVERSATION_URL.format(conversation_id)}) '- 'for more information.'- )- except Exception as post_error:- _logger.warning(- '[GitLab V1] Failed to post error message to GitLab: %s', post_error- )+ can_post_error = bool(self.gitlab_view_data.get('keycloak_user_id'))+ await handle_callback_error(+ error=e,+ conversation_id=conversation_id,+ service_name='GitLab',+ service_logger=_logger,+ can_post_error=can_post_error,+ post_error_func=self._post_summary_to_gitlab,+ )return EventCallbackResult(status=EventCallbackResultStatus.ERROR,@@ -149,8 +145,8 @@ class GitlabV1CallbackProcessor(EventCallbackProcessor):send_message_request = AskAgentRequest(question=message_content)url = (- f'{agent_server_url.rstrip("/")}'- f'/api/conversations/{conversation_id}/ask_agent'+ f"{agent_server_url.rstrip('/')}"+ f"/api/conversations/{conversation_id}/ask_agent")headers = {'X-Session-API-Key': session_api_key}payload = send_message_request.model_dump()diff --git a/enterprise/integrations/slack/slack_v1_callback_processor.py b/enterprise/integrations/slack/slack_v1_callback_processor.pyindex e5724b1df5..25b94b7872 100644--- a/enterprise/integrations/slack/slack_v1_callback_processor.py+++ b/enterprise/integrations/slack/slack_v1_callback_processor.py@@ -2,7 +2,8 @@ import loggingfrom uuid import UUIDimport httpx-from integrations.utils import CONVERSATION_URL, get_summary_instruction+from integrations.utils import get_summary_instruction+from integrations.v1_utils import handle_callback_errorfrom pydantic import Fieldfrom slack_sdk import WebClientfrom storage.slack_team_store import SlackTeamStore@@ -39,7 +40,6 @@ class SlackV1CallbackProcessor(EventCallbackProcessor):event: Event,) -> EventCallbackResult | None:"""Process events for Slack V1 integration."""-# Only handle ConversationStateUpdateEventif not isinstance(event, ConversationStateUpdateEvent):return None@@ -62,19 +62,14 @@ class SlackV1CallbackProcessor(EventCallbackProcessor):detail=summary,)except Exception as e:- _logger.exception('[Slack V1] Error processing callback: %s', e)-- # Only try to post error to Slack if we have basic requirements- try:- await self._post_summary_to_slack(- f'OpenHands encountered an error: **{str(e)}**.\n\n'- f'[See the conversation]({CONVERSATION_URL.format(conversation_id)})'- 'for more information.'- )- except Exception as post_error:- _logger.warning(- '[Slack V1] Failed to post error message to Slack: %s', post_error- )+ await handle_callback_error(+ error=e,+ conversation_id=conversation_id,+ service_name='Slack',+ service_logger=_logger,+ can_post_error=True, # Slack always attempts to post errors+ post_error_func=self._post_summary_to_slack,+ )return EventCallbackResult(status=EventCallbackResultStatus.ERROR,@@ -149,8 +144,8 @@ class SlackV1CallbackProcessor(EventCallbackProcessor):send_message_request = AskAgentRequest(question=message_content)url = (- f'{agent_server_url.rstrip("/")}'- f'/api/conversations/{conversation_id}/ask_agent'+ f"{agent_server_url.rstrip('/')}"+ f"/api/conversations/{conversation_id}/ask_agent")headers = {'X-Session-API-Key': session_api_key}payload = send_message_request.model_dump()@@ -212,8 +207,7 @@ class SlackV1CallbackProcessor(EventCallbackProcessor):# -------------------------------------------------------------------------async def _request_summary(self, conversation_id: UUID) -> str:- """- Ask the agent to produce a summary of its work and return the agent response.+ """Ask the agent to produce a summary of its work and return the agent response.NOTE: This method now returns a string (the agent server's response text)and raises exceptions on errors. The wrapping into EventCallbackResultdiff --git a/enterprise/integrations/v1_utils.py b/enterprise/integrations/v1_utils.pyindex 78953e4e93..1b42fdd567 100644--- a/enterprise/integrations/v1_utils.py+++ b/enterprise/integrations/v1_utils.py@@ -1,3 +1,8 @@+import logging+from typing import Callable, Coroutine+from uuid import UUID++from integrations.utils import CONVERSATION_URLfrom pydantic import SecretStrfrom server.auth.saas_user_auth import SaasUserAuthfrom server.auth.token_manager import TokenManager@@ -6,6 +11,78 @@ from openhands.core.logger import openhands_logger as loggerfrom openhands.server.user_auth.user_auth import UserAuth+def is_budget_exceeded_error(error_message: str) -> bool:+ """Check if an error message indicates a budget exceeded condition.++ This is used to downgrade error logs to info logs for budget exceeded errors+ since they are expected cost control behavior rather than unexpected errors.+ """+ lower_message = error_message.lower()+ return 'budget' in lower_message and 'exceeded' in lower_message+++BUDGET_EXCEEDED_USER_MESSAGE = 'LLM budget has been exceeded, please re-fill.'+++async def handle_callback_error(+ error: Exception,+ conversation_id: UUID,+ service_name: str,+ service_logger: logging.Logger,+ can_post_error: bool,+ post_error_func: Callable[[str], Coroutine],+) -> None:+ """Handle callback processing errors with appropriate logging and user messages.++ This centralizes the error handling logic for V1 callback processors to:+ - Log budget exceeded errors at INFO level (expected cost control behavior)+ - Log other errors at EXCEPTION level+ - Post user-friendly error messages to the integration platform++ Args:+ error: The exception that occurred+ conversation_id: The conversation ID for logging and linking+ service_name: The service name for log messages (e.g., "GitHub", "GitLab", "Slack")+ service_logger: The logger instance to use for logging+ can_post_error: Whether the prerequisites are met to post an error message+ post_error_func: Async function to post the error message to the platform+ """+ error_str = str(error)+ budget_exceeded = is_budget_exceeded_error(error_str)++ # Log appropriately based on error type+ if budget_exceeded:+ service_logger.info(+ '[%s V1] Budget exceeded for conversation %s: %s',+ service_name,+ conversation_id,+ error,+ )+ else:+ service_logger.exception(+ '[%s V1] Error processing callback: %s', service_name, error+ )++ # Try to post error message to the platform+ if can_post_error:+ try:+ error_detail = (+ BUDGET_EXCEEDED_USER_MESSAGE if budget_exceeded else error_str+ )+ await post_error_func(+ f'OpenHands encountered an error: **{error_detail}**\n\n'+ f'[See the conversation]({CONVERSATION_URL.format(conversation_id)}) '+ 'for more information.'+ )+ except Exception as post_error:+ service_logger.warning(+ '[%s V1] Failed to post error message to %s: %s',+ service_name,+ service_name,+ post_error,+ )++async def get_saas_user_auth(keycloak_user_id: str, token_manager: TokenManager) -> UserAuth:diff --git a/enterprise/tests/unit/integrations/github/test_github_v1_callback_processor.py b/enterprise/tests/unit/integrations/github/test_github_v1_callback_processor.pyindex 1205feaba6..36ada82fe8 100644--- a/enterprise/tests/unit/integrations/github/test_github_v1_callback_processor.py+++ b/enterprise/tests/unit/integrations/github/test_github_v1_callback_processor.py@@ -809,3 +809,94 @@ class TestGithubV1CallbackProcessor:)assert f'conversations/{conversation_id}' in error_commentassert 'for more information.' in error_comment++ @patch(+ 'integrations.github.github_v1_callback_processor.GITHUB_APP_CLIENT_ID',+ 'test_client_id',+ )+ @patch(+ 'integrations.github.github_v1_callback_processor.GITHUB_APP_PRIVATE_KEY',+ 'test_private_key',+ )+ @patch('integrations.github.github_v1_callback_processor.get_summary_instruction')+ @patch('openhands.app_server.config.get_httpx_client')+ @patch('openhands.app_server.config.get_sandbox_service')+ @patch('openhands.app_server.config.get_app_conversation_info_service')+ @patch('integrations.github.github_v1_callback_processor._logger')+ async def test_budget_exceeded_error_logs_info_and_sends_friendly_message(+ self,+ mock_logger,+ mock_get_app_conversation_info_service,+ mock_get_sandbox_service,+ mock_get_httpx_client,+ mock_get_summary_instruction,+ github_callback_processor,+ conversation_state_update_event,+ event_callback,+ mock_app_conversation_info,+ mock_sandbox_info,+ ):+ """Test that budget exceeded errors are logged at INFO level and user gets friendly message."""+ conversation_id = uuid4()++ mock_httpx_client = await _setup_happy_path_services(+ mock_get_app_conversation_info_service,+ mock_get_sandbox_service,+ mock_get_httpx_client,+ mock_app_conversation_info,+ mock_sandbox_info,+ )+ # Simulate a budget exceeded error from the agent server+ budget_error_msg = (+ 'HTTP 500 error: {"detail":"Internal Server Error",'+ '"exception":"litellm.BadRequestError: Litellm_proxyException - '+ 'Budget has been exceeded! Current cost: 12.65, Max budget: 12.62"}'+ )+ mock_httpx_client.post.side_effect = Exception(budget_error_msg)+ mock_get_summary_instruction.return_value = 'Please provide a summary'++ with (+ patch(+ 'integrations.github.github_v1_callback_processor.GithubIntegration'+ ) as mock_github_integration,+ patch(+ 'integrations.github.github_v1_callback_processor.Github'+ ) as mock_github,+ ):+ mock_integration = MagicMock()+ mock_github_integration.return_value = mock_integration+ mock_integration.get_access_token.return_value.token = 'test_token'++ mock_gh = MagicMock()+ mock_github.return_value.__enter__.return_value = mock_gh+ mock_repo = MagicMock()+ mock_issue = MagicMock()+ mock_repo.get_issue.return_value = mock_issue+ mock_gh.get_repo.return_value = mock_repo++ result = await github_callback_processor(+ conversation_id=conversation_id,+ callback=event_callback,+ event=conversation_state_update_event,+ )++ assert result is not None+ assert result.status == EventCallbackResultStatus.ERROR++ # Verify exception was NOT called (budget exceeded uses info instead)+ mock_logger.exception.assert_not_called()++ # Verify budget exceeded info log was called+ info_calls = [str(call) for call in mock_logger.info.call_args_list]+ budget_log_found = any('Budget exceeded' in call for call in info_calls)+ assert budget_log_found, f'Expected budget exceeded log, got: {info_calls}'++ # Verify user-friendly message was posted to GitHub+ mock_issue.create_comment.assert_called_once()+ call_args = mock_issue.create_comment.call_args+ posted_comment = call_args[1].get('body') or call_args[0][0]+ assert 'OpenHands encountered an error' in posted_comment+ assert 'LLM budget has been exceeded' in posted_comment+ assert 'please re-fill' in posted_comment+ # Should NOT contain the raw error message+ assert 'litellm.BadRequestError' not in posted_commentdiff --git a/enterprise/tests/unit/integrations/gitlab/test_gitlab_v1_callback_processor.py b/enterprise/tests/unit/integrations/gitlab/test_gitlab_v1_callback_processor.pyindex 252d208e08..43fab5d862 100644--- a/enterprise/tests/unit/integrations/gitlab/test_gitlab_v1_callback_processor.py+++ b/enterprise/tests/unit/integrations/gitlab/test_gitlab_v1_callback_processor.py@@ -374,3 +374,72 @@ class TestGitlabV1CallbackProcessor:assert 'OpenHands encountered an error' in error_commentassert 'Simulated agent server error' in error_commentassert f'conversations/{conversation_id}' in error_comment++ @patch('openhands.app_server.config.get_app_conversation_info_service')+ @patch('openhands.app_server.config.get_sandbox_service')+ @patch('openhands.app_server.config.get_httpx_client')+ @patch('integrations.gitlab.gitlab_v1_callback_processor.get_summary_instruction')+ @patch('integrations.gitlab.gitlab_service.SaaSGitLabService')+ @patch('integrations.gitlab.gitlab_v1_callback_processor._logger')+ async def test_budget_exceeded_error_logs_info_and_sends_friendly_message(+ self,+ mock_logger,+ mock_saas_gitlab_service_cls,+ mock_get_summary_instruction,+ mock_get_httpx_client,+ mock_get_sandbox_service,+ mock_get_app_conversation_info_service,+ gitlab_callback_processor,+ conversation_state_update_event,+ event_callback,+ mock_app_conversation_info,+ mock_sandbox_info,+ ):+ """Test that budget exceeded errors are logged at INFO level and user gets friendly message."""+ conversation_id = uuid4()++ mock_httpx_client = await _setup_happy_path_services(+ mock_get_app_conversation_info_service,+ mock_get_sandbox_service,+ mock_get_httpx_client,+ mock_app_conversation_info,+ mock_sandbox_info,+ )+ # Simulate a budget exceeded error from the agent server+ budget_error_msg = (+ 'HTTP 500 error: {"detail":"Internal Server Error",'+ '"exception":"litellm.BadRequestError: Litellm_proxyException - '+ 'Budget has been exceeded! Current cost: 12.65, Max budget: 12.62"}'+ )+ mock_httpx_client.post.side_effect = Exception(budget_error_msg)+ mock_get_summary_instruction.return_value = 'Please provide a summary'++ mock_gitlab_service = AsyncMock()+ mock_saas_gitlab_service_cls.return_value = mock_gitlab_service++ result = await gitlab_callback_processor(+ conversation_id=conversation_id,+ callback=event_callback,+ event=conversation_state_update_event,+ )++ assert result is not None+ assert result.status == EventCallbackResultStatus.ERROR++ # Verify exception was NOT called (budget exceeded uses info instead)+ mock_logger.exception.assert_not_called()++ # Verify budget exceeded info log was called+ info_calls = [str(call) for call in mock_logger.info.call_args_list]+ budget_log_found = any('Budget exceeded' in call for call in info_calls)+ assert budget_log_found, f'Expected budget exceeded log, got: {info_calls}'++ # Verify user-friendly message was posted to GitLab+ mock_gitlab_service.reply_to_issue.assert_called_once()+ call_args = mock_gitlab_service.reply_to_issue.call_args+ posted_comment = call_args[0][3] # 4th positional arg is the body+ assert 'OpenHands encountered an error' in posted_comment+ assert 'LLM budget has been exceeded' in posted_comment+ assert 'please re-fill' in posted_comment+ # Should NOT contain the raw error message+ assert 'litellm.BadRequestError' not in posted_commentdiff --git a/enterprise/tests/unit/integrations/slack/test_slack_v1_callback_processor.py b/enterprise/tests/unit/integrations/slack/test_slack_v1_callback_processor.pyindex e72e89233e..209507c64b 100644--- a/enterprise/tests/unit/integrations/slack/test_slack_v1_callback_processor.py+++ b/enterprise/tests/unit/integrations/slack/test_slack_v1_callback_processor.py@@ -429,3 +429,89 @@ class TestSlackV1CallbackProcessor:assert result is not Noneassert result.status == EventCallbackResultStatus.ERRORassert expected_error_fragment in result.detail++ @patch('storage.slack_team_store.SlackTeamStore.get_instance')+ @patch('openhands.app_server.config.get_httpx_client')+ @patch('openhands.app_server.config.get_sandbox_service')+ @patch('openhands.app_server.config.get_app_conversation_info_service')+ @patch('integrations.slack.slack_v1_callback_processor.get_summary_instruction')+ @patch('integrations.slack.slack_v1_callback_processor._logger')+ @patch('integrations.slack.slack_v1_callback_processor.WebClient')+ async def test_budget_exceeded_error_logs_info_and_sends_friendly_message(+ self,+ mock_web_client_cls,+ mock_logger,+ mock_get_summary_instruction,+ mock_get_app_conversation_info_service,+ mock_get_sandbox_service,+ mock_get_httpx_client,+ mock_slack_team_store,+ slack_callback_processor,+ finish_event,+ event_callback,+ mock_app_conversation_info,+ mock_sandbox_info,+ ):+ """Test that budget exceeded errors are logged at INFO level and user gets friendly message."""+ conversation_id = uuid4()++ # Mock SlackTeamStore+ mock_store = MagicMock()+ mock_store.get_team_bot_token = AsyncMock(return_value='xoxb-test-token')+ mock_slack_team_store.return_value = mock_store++ mock_get_summary_instruction.return_value = 'Please provide a summary'++ # Mock services+ mock_app_conversation_info_service = AsyncMock()+ mock_app_conversation_info_service.get_app_conversation_info.return_value = (+ mock_app_conversation_info+ )+ mock_get_app_conversation_info_service.return_value.__aenter__.return_value = (+ mock_app_conversation_info_service+ )++ mock_sandbox_service = AsyncMock()+ mock_sandbox_service.get_sandbox.return_value = mock_sandbox_info+ mock_get_sandbox_service.return_value.__aenter__.return_value = (+ mock_sandbox_service+ )++ # Simulate a budget exceeded error from the agent server+ budget_error_msg = (+ 'HTTP 500 error: {"detail":"Internal Server Error",'+ '"exception":"litellm.BadRequestError: Litellm_proxyException - '+ 'Budget has been exceeded! Current cost: 12.65, Max budget: 12.62"}'+ )+ mock_httpx_client = AsyncMock()+ mock_httpx_client.post.side_effect = Exception(budget_error_msg)+ mock_get_httpx_client.return_value.__aenter__.return_value = mock_httpx_client++ # Mock Slack WebClient+ mock_slack_client = MagicMock()+ mock_web_client_cls.return_value = mock_slack_client++ result = await slack_callback_processor(+ conversation_id, event_callback, finish_event+ )++ assert result is not None+ assert result.status == EventCallbackResultStatus.ERROR++ # Verify exception was NOT called (budget exceeded uses info instead)+ mock_logger.exception.assert_not_called()++ # Verify budget exceeded info log was called+ info_calls = [str(call) for call in mock_logger.info.call_args_list]+ budget_log_found = any('Budget exceeded' in call for call in info_calls)+ assert budget_log_found, f'Expected budget exceeded log, got: {info_calls}'++ # Verify user-friendly message was posted to Slack+ mock_slack_client.chat_postMessage.assert_called_once()+ call_kwargs = mock_slack_client.chat_postMessage.call_args[1]+ posted_message = call_kwargs.get('text', '')+ assert 'OpenHands encountered an error' in posted_message+ assert 'LLM budget has been exceeded' in posted_message+ assert 'please re-fill' in posted_message+ # Should NOT contain the raw error message+ assert 'litellm.BadRequestError' not in posted_message