Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Modify experiment manager defaults for nested runtimes (#10269)

Rohit Malhotra368a0248e33c
diff --git a/openhands/experiments/experiment_manager.py b/openhands/experiments/experiment_manager.py
index 2e84542e9a..96a81b9652 100644
--- a/openhands/experiments/experiment_manager.py
+++ b/openhands/experiments/experiment_manager.py
@@ -1,11 +1,33 @@
import os
+from pydantic import BaseModel
+
from openhands.core.config.openhands_config import OpenHandsConfig
from openhands.core.logger import openhands_logger as logger
from openhands.server.session.conversation_init_data import ConversationInitData
+from openhands.server.shared import file_store
+from openhands.storage.locations import get_experiment_config_filename
from openhands.utils.import_utils import get_impl
+class ExperimentConfig(BaseModel):
+ config: dict[str, str] | None = None
+
+
+def load_experiment_config(conversation_id: str) -> ExperimentConfig | None:
+ try:
+ file_path = get_experiment_config_filename(conversation_id)
+ exp_config = file_store.read(file_path)
+ return ExperimentConfig.model_validate_json(exp_config)
+
+ except FileNotFoundError:
+ pass
+ except Exception as e:
+ logger.warning(f'Failed to load experiment config: {e}')
+
+ return None
+
+
class ExperimentManager:
@staticmethod
def run_conversation_variant_test(
@@ -17,9 +39,19 @@ class ExperimentManager:
def run_config_variant_test(
user_id: str, conversation_id: str, config: OpenHandsConfig
) -> OpenHandsConfig:
- logger.debug(
- f'Running agent config variant test for user_id={user_id}, conversation_id={conversation_id}'
- )
+ exp_config = load_experiment_config(conversation_id)
+ if exp_config and exp_config.config:
+ agent_cfg = config.get_agent_config(config.default_agent)
+ try:
+ for attr, value in exp_config.config.items():
+ if hasattr(agent_cfg, attr):
+ logger.info(
+ f'Set attrib {attr} to {value} for {conversation_id}'
+ )
+ setattr(agent_cfg, attr, value)
+ except Exception as e:
+ logger.warning(f'Error processing exp config: {e}')
+
return config
diff --git a/openhands/server/constants.py b/openhands/server/constants.py
new file mode 100644
index 0000000000..2f09bab748
--- /dev/null
+++ b/openhands/server/constants.py
@@ -0,0 +1,3 @@
+"""Server constants."""
+
+ROOM_KEY = 'room:{sid}'
diff --git a/openhands/server/conversation_manager/docker_nested_conversation_manager.py b/openhands/server/conversation_manager/docker_nested_conversation_manager.py
index 275e9a4a8d..82db976a4f 100644
--- a/openhands/server/conversation_manager/docker_nested_conversation_manager.py
+++ b/openhands/server/conversation_manager/docker_nested_conversation_manager.py
@@ -25,6 +25,7 @@ from openhands.llm.llm import LLM
from openhands.runtime import get_runtime_cls
from openhands.runtime.impl.docker.docker_runtime import DockerRuntime
from openhands.server.config.server_config import ServerConfig
+from openhands.server.constants import ROOM_KEY
from openhands.server.conversation_manager.conversation_manager import (
ConversationManager,
)
@@ -32,7 +33,7 @@ from openhands.server.data_models.agent_loop_info import AgentLoopInfo
from openhands.server.monitoring import MonitoringListener
from openhands.server.session.conversation import ServerConversation
from openhands.server.session.conversation_init_data import ConversationInitData
-from openhands.server.session.session import ROOM_KEY, Session
+from openhands.server.session.session import Session
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.conversation_metadata import ConversationMetadata
from openhands.storage.data_models.conversation_status import ConversationStatus
diff --git a/openhands/server/conversation_manager/standalone_conversation_manager.py b/openhands/server/conversation_manager/standalone_conversation_manager.py
index ad8dc5b0b5..b646d40f70 100644
--- a/openhands/server/conversation_manager/standalone_conversation_manager.py
+++ b/openhands/server/conversation_manager/standalone_conversation_manager.py
@@ -14,11 +14,12 @@ from openhands.events.action import MessageAction
from openhands.events.stream import EventStreamSubscriber, session_exists
from openhands.runtime import get_runtime_cls
from openhands.server.config.server_config import ServerConfig
+from openhands.server.constants import ROOM_KEY
from openhands.server.data_models.agent_loop_info import AgentLoopInfo
from openhands.server.monitoring import MonitoringListener
from openhands.server.session.agent_session import WAIT_TIME_BEFORE_CLOSE, AgentSession
from openhands.server.session.conversation import ServerConversation
-from openhands.server.session.session import ROOM_KEY, Session
+from openhands.server.session.session import Session
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.conversation_metadata import ConversationMetadata
from openhands.storage.data_models.conversation_status import ConversationStatus
diff --git a/openhands/server/routes/manage_conversations.py b/openhands/server/routes/manage_conversations.py
index 4d5dbc077d..4da6b7e310 100644
--- a/openhands/server/routes/manage_conversations.py
+++ b/openhands/server/routes/manage_conversations.py
@@ -22,6 +22,7 @@ from openhands.events.observation import (
AgentStateChangedObservation,
NullObservation,
)
+from openhands.experiments.experiment_manager import ExperimentConfig
from openhands.integrations.provider import (
PROVIDER_TOKEN_TYPE,
ProviderHandler,
@@ -71,6 +72,7 @@ from openhands.storage.data_models.conversation_metadata import (
from openhands.storage.data_models.conversation_status import ConversationStatus
from openhands.storage.data_models.settings import Settings
from openhands.storage.data_models.user_secrets import UserSecrets
+from openhands.storage.locations import get_experiment_config_filename
from openhands.storage.settings.settings_store import SettingsStore
from openhands.utils.async_utils import wait_all
from openhands.utils.conversation_summary import get_default_conversation_title
@@ -707,3 +709,28 @@ async def update_conversation(
},
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
+
+
[email protected]('/conversations/{conversation_id}/exp-config')
+def add_experiment_config_for_conversation(
+ conversation_id: str, exp_config: ExperimentConfig
+) -> bool:
+ exp_config_filepath = get_experiment_config_filename(conversation_id)
+ exists = False
+ try:
+ file_store.read(exp_config_filepath)
+ exists = True
+ except FileNotFoundError:
+ pass
+
+ # Don't modify again if it already exists
+ if exists:
+ return False
+
+ try:
+ file_store.write(exp_config_filepath, exp_config.model_dump_json())
+ except Exception as e:
+ logger.info(f'Failed to write experiment config for {conversation_id}: {e}')
+ return True
+
+ return False
diff --git a/openhands/server/session/session.py b/openhands/server/session/session.py
index 5f2836322e..458f45abe2 100644
--- a/openhands/server/session/session.py
+++ b/openhands/server/session/session.py
@@ -28,16 +28,14 @@ from openhands.events.observation.agent import RecallObservation
from openhands.events.observation.error import ErrorObservation
from openhands.events.serialization import event_from_dict, event_to_dict
from openhands.events.stream import EventStreamSubscriber
-from openhands.experiments.experiment_manager import ExperimentManagerImpl
from openhands.llm.llm import LLM
from openhands.runtime.runtime_status import RuntimeStatus
+from openhands.server.constants import ROOM_KEY
from openhands.server.session.agent_session import AgentSession
from openhands.server.session.conversation_init_data import ConversationInitData
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
-ROOM_KEY = 'room:{sid}'
-
class Session:
sid: str
@@ -75,6 +73,9 @@ class Session:
)
# Copying this means that when we update variables they are not applied to the shared global configuration!
self.config = deepcopy(config)
+ # Lazy import to avoid circular dependency
+ from openhands.experiments.experiment_manager import ExperimentManagerImpl
+
self.config = ExperimentManagerImpl.run_config_variant_test(
user_id, sid, self.config
)
diff --git a/openhands/storage/locations.py b/openhands/storage/locations.py
index 43e1661b91..192721da86 100644
--- a/openhands/storage/locations.py
+++ b/openhands/storage/locations.py
@@ -28,3 +28,7 @@ def get_conversation_init_data_filename(sid: str, user_id: str | None = None) ->
def get_conversation_agent_state_filename(sid: str, user_id: str | None = None) -> str:
return f'{get_conversation_dir(sid, user_id)}agent_state.pkl'
+
+
+def get_experiment_config_filename(sid: str, user_id: str | None = None) -> str:
+ return f'{get_conversation_dir(sid, user_id)}exp_config.json'