Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Add litellm_extra_body metadata for V1 conversations (#12266)

Xingyao Wanga8098505c291
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
index 887c4dfeb4..ac4609e648 100644
--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py
+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
@@ -72,6 +72,10 @@ from openhands.app_server.user.user_models import UserInfo
from openhands.app_server.utils.docker_utils import (
replace_localhost_hostname_for_docker,
)
+from openhands.app_server.utils.llm_metadata import (
+ get_llm_metadata,
+ should_set_litellm_extra_body,
+)
from openhands.experiments.experiment_manager import ExperimentManagerImpl
from openhands.integrations.provider import ProviderType
from openhands.sdk import Agent, AgentContext, LocalWorkspace
@@ -892,6 +896,63 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
return agent
+ def _update_agent_with_llm_metadata(
+ self,
+ agent: Agent,
+ conversation_id: UUID,
+ user_id: str | None,
+ ) -> Agent:
+ """Update agent's LLM and condenser LLM with litellm_extra_body metadata.
+
+ This adds tracing metadata (conversation_id, user_id, etc.) to the LLM
+ for analytics and debugging purposes. Only applies to openhands/ models.
+
+ Args:
+ agent: The agent to update
+ conversation_id: The conversation ID
+ user_id: The user ID (can be None)
+
+ Returns:
+ Updated agent with LLM metadata
+ """
+ updates: dict[str, Any] = {}
+
+ # Update main LLM if it's an openhands model
+ if should_set_litellm_extra_body(agent.llm.model):
+ llm_metadata = get_llm_metadata(
+ model_name=agent.llm.model,
+ llm_type=agent.llm.usage_id or 'agent',
+ conversation_id=conversation_id,
+ user_id=user_id,
+ )
+ updated_llm = agent.llm.model_copy(
+ update={'litellm_extra_body': {'metadata': llm_metadata}}
+ )
+ updates['llm'] = updated_llm
+
+ # Update condenser LLM if it exists and is an openhands model
+ if agent.condenser and hasattr(agent.condenser, 'llm'):
+ condenser_llm = agent.condenser.llm
+ if should_set_litellm_extra_body(condenser_llm.model):
+ condenser_metadata = get_llm_metadata(
+ model_name=condenser_llm.model,
+ llm_type=condenser_llm.usage_id or 'condenser',
+ conversation_id=conversation_id,
+ user_id=user_id,
+ )
+ updated_condenser_llm = condenser_llm.model_copy(
+ update={'litellm_extra_body': {'metadata': condenser_metadata}}
+ )
+ updated_condenser = agent.condenser.model_copy(
+ update={'llm': updated_condenser_llm}
+ )
+ updates['condenser'] = updated_condenser
+
+ # Return updated agent if there are changes
+ if updates:
+ return agent.model_copy(update=updates)
+ return agent
+
async def _finalize_conversation_request(
self,
agent: Agent,
@@ -930,6 +991,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
user.id, conversation_id, agent
)
+ # Update agent's LLM with litellm_extra_body metadata for tracing
+ # This is done after experiment variants to ensure the final LLM config is used
+ agent = self._update_agent_with_llm_metadata(agent, conversation_id, user.id)
+
# Load and merge skills if remote workspace is available
if remote_workspace:
try:
diff --git a/openhands/app_server/utils/llm_metadata.py b/openhands/app_server/utils/llm_metadata.py
new file mode 100644
index 0000000000..8413b437e3
--- /dev/null
+++ b/openhands/app_server/utils/llm_metadata.py
@@ -0,0 +1,73 @@
+"""Utility functions for LLM metadata in OpenHands V1 conversations."""
+
+import os
+from typing import Any
+from uuid import UUID
+
+import openhands
+
+
+def should_set_litellm_extra_body(model_name: str) -> bool:
+ """Determine if litellm_extra_body should be set based on the model name.
+
+ Only set litellm_extra_body for openhands models to avoid issues
+ with providers that don't support extra_body parameters.
+
+ The SDK internally translates "openhands/" prefix to "litellm_proxy/"
+ when making API calls, so we check for both.
+
+ Args:
+ model_name: Name of the LLM model
+
+ Returns:
+ True if litellm_extra_body should be set, False otherwise
+ """
+ return 'openhands/' in model_name or 'litellm_proxy/' in model_name
+
+
+def get_llm_metadata(
+ model_name: str,
+ llm_type: str,
+ conversation_id: UUID | str | None = None,
+ user_id: str | None = None,
+) -> dict[str, Any]:
+ """Generate LLM metadata for OpenHands V1 conversations.
+
+ This metadata is passed to the LiteLLM proxy for tracing and analytics.
+
+ Args:
+ model_name: Name of the LLM model
+ llm_type: Type of LLM usage (e.g., 'agent', 'condenser', 'planning_condenser')
+ conversation_id: Optional conversation identifier
+ user_id: Optional user identifier
+
+ Returns:
+ Dictionary containing metadata for LLM initialization
+ """
+ openhands_version = openhands.__version__
+
+ metadata: dict[str, Any] = {
+ 'trace_version': openhands_version,
+ 'tags': [
+ 'app:openhands',
+ f'model:{model_name}',
+ f'type:{llm_type}',
+ f'web_host:{os.environ.get("WEB_HOST", "unspecified")}',
+ f'openhands_version:{openhands_version}',
+ 'conversation_version:V1',
+ ],
+ }
+
+ if conversation_id is not None:
+ # Convert UUID to string if needed
+ session_id = (
+ str(conversation_id)
+ if isinstance(conversation_id, UUID)
+ else conversation_id
+ )
+ metadata['session_id'] = session_id
+
+ if user_id is not None:
+ metadata['trace_user_id'] = user_id
+
+ return metadata
diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py
index 126e54c69c..f37a7894b8 100644
--- a/tests/unit/app_server/test_live_status_app_conversation_service.py
+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py
@@ -793,7 +793,15 @@ class TestLiveStatusAppConversationService:
"""Test _finalize_conversation_request with skills loading."""
# Arrange
mock_agent = Mock(spec=Agent)
+
+ # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+ mock_llm = Mock(spec=LLM)
+ mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update
+ mock_llm.usage_id = 'agent'
+
mock_updated_agent = Mock(spec=Agent)
+ mock_updated_agent.llm = mock_llm
+ mock_updated_agent.condenser = None # No condenser
mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
mock_updated_agent
)
@@ -852,7 +860,15 @@ class TestLiveStatusAppConversationService:
"""Test _finalize_conversation_request without remote workspace (no skills)."""
# Arrange
mock_agent = Mock(spec=Agent)
+
+ # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+ mock_llm = Mock(spec=LLM)
+ mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update
+ mock_llm.usage_id = 'agent'
+
mock_updated_agent = Mock(spec=Agent)
+ mock_updated_agent.llm = mock_llm
+ mock_updated_agent.condenser = None # No condenser
mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
mock_updated_agent
)
@@ -890,7 +906,15 @@ class TestLiveStatusAppConversationService:
"""Test _finalize_conversation_request when skills loading fails."""
# Arrange
mock_agent = Mock(spec=Agent)
+
+ # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+ mock_llm = Mock(spec=LLM)
+ mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update
+ mock_llm.usage_id = 'agent'
+
mock_updated_agent = Mock(spec=Agent)
+ mock_updated_agent.llm = mock_llm
+ mock_updated_agent.condenser = None # No condenser
mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
mock_updated_agent
)
diff --git a/tests/unit/experiments/test_experiment_manager.py b/tests/unit/experiments/test_experiment_manager.py
index 70cd6c5d07..6a9d5e4a32 100644
--- a/tests/unit/experiments/test_experiment_manager.py
+++ b/tests/unit/experiments/test_experiment_manager.py
@@ -139,6 +139,7 @@ class TestExperimentManagerIntegration:
mock_agent = Mock(spec=Agent)
mock_agent.llm = mock_llm
+ mock_agent.condenser = None # No condenser for this test
mock_agent.system_prompt_filename = 'default_system_prompt.j2'
mock_agent.model_copy = Mock(return_value=mock_agent)