π OpenHands: AI-Driven Development
[Feat]: Add experiment manager (#8820)
Rohit Malhotra93b127676828
diff --git a/openhands/experiments/experiment_manager.py b/openhands/experiments/experiment_manager.pynew file mode 100644index 0000000000..39c6af6284--- /dev/null+++ b/openhands/experiments/experiment_manager.py@@ -0,0 +1,19 @@+import os++from openhands.server.session.conversation_init_data import ConversationInitData+from openhands.utils.import_utils import get_impl+++class ExperimentManager:+ @staticmethod+ def run_conversation_variant_test(+ user_id: str, conversation_id: str, conversation_settings: ConversationInitData+ ) -> ConversationInitData:+ return conversation_settings+++experiment_manager_cls = os.environ.get(+ 'OPENHANDS_EXPERIMENT_MANAGER_CLS',+ 'openhands.experiments.experiment_manager.ExperimentManager',+)+ExperimentManagerImpl = get_impl(ExperimentManager, experiment_manager_cls)diff --git a/openhands/server/listen_socket.py b/openhands/server/listen_socket.pyindex 365c63d1d0..dd49f30bce 100644--- a/openhands/server/listen_socket.py+++ b/openhands/server/listen_socket.py@@ -19,6 +19,7 @@ from openhands.events.observation.agent import (AgentStateChangedObservation,)from openhands.events.serialization import event_to_dict+from openhands.experiments.experiment_manager import ExperimentManagerImplfrom openhands.integrations.provider import PROVIDER_TOKEN_TYPE, ProviderTokenfrom openhands.integrations.service_types import ProviderTypefrom openhands.server.session.conversation_init_data import ConversationInitData@@ -49,7 +50,7 @@ def create_provider_tokens_object(async def setup_init_convo_settings(- user_id: str | None, providers_set: list[ProviderType]+ user_id: str | None, conversation_id: str, providers_set: list[ProviderType]) -> ConversationInitData:settings_store = await SettingsStoreImpl.get_instance(config, user_id)settings = await settings_store.load()@@ -73,7 +74,11 @@ async def setup_init_convo_settings(if user_secrets:session_init_args['custom_secrets'] = user_secrets.custom_secrets- return ConversationInitData(**session_init_args)+ convo_init_data = ConversationInitData(**session_init_args)+ # We should recreate the same experiment conditions when restarting a conversation+ return ExperimentManagerImpl.run_conversation_variant_test(+ user_id, conversation_id, convo_init_data+ )@sio.event@@ -119,7 +124,9 @@ async def connect(connection_id: str, environ: dict) -> None:f'User {user_id} is allowed to connect to conversation {conversation_id}')- conversation_init_data = await setup_init_convo_settings(user_id, providers_set)+ conversation_init_data = await setup_init_convo_settings(+ user_id, conversation_id, providers_set+ )agent_loop_info = await conversation_manager.join_conversation(conversation_id,connection_id,diff --git a/openhands/server/services/conversation_service.py b/openhands/server/services/conversation_service.pyindex e0e9867d4c..ccb3c75e90 100644--- a/openhands/server/services/conversation_service.py+++ b/openhands/server/services/conversation_service.py@@ -4,6 +4,7 @@ from typing import Anyfrom openhands.core.logger import openhands_logger as loggerfrom openhands.events.action.message import MessageAction+from openhands.experiments.experiment_manager import ExperimentManagerImplfrom openhands.integrations.provider import (CUSTOM_SECRETS_TYPE_WITH_JSON_SCHEMA,PROVIDER_TOKEN_TYPE,@@ -78,6 +79,8 @@ async def create_new_conversation(session_init_args['git_provider'] = git_providersession_init_args['conversation_instructions'] = conversation_instructionsconversation_init_data = ConversationInitData(**session_init_args)++logger.info('Loading conversation store')conversation_store = await ConversationStoreImpl.get_instance(config, user_id)logger.info('ServerConversation store loaded')@@ -93,6 +96,7 @@ async def create_new_conversation(extra={'user_id': user_id, 'session_id': conversation_id},)+ conversation_init_data = ExperimentManagerImpl.run_conversation_variant_test(user_id, conversation_id, conversation_init_data)conversation_title = get_default_conversation_title(conversation_id)logger.info(f'Saving metadata for conversation {conversation_id}')@@ -105,6 +109,7 @@ async def create_new_conversation(selected_repository=selected_repository,selected_branch=selected_branch,git_provider=git_provider,+ llm_model=settings.llm_model,))diff --git a/openhands/storage/data_models/conversation_metadata.py b/openhands/storage/data_models/conversation_metadata.pyindex d089811afa..8adea2835a 100644--- a/openhands/storage/data_models/conversation_metadata.py+++ b/openhands/storage/data_models/conversation_metadata.py@@ -25,6 +25,7 @@ class ConversationMetadata:trigger: ConversationTrigger | None = Nonepr_number: list[int] = field(default_factory=list)created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))+ llm_model: str | None = None# Cost and token metricsaccumulated_cost: float = 0.0prompt_tokens: int = 0