π OpenHands: AI-Driven Development
Removed V0 conversation stats (#14152)
Tim O'Farrellfc4c62a73db3
diff --git a/enterprise/server/utils/conversation_callback_utils.py b/enterprise/server/utils/conversation_callback_utils.pyindex 5593448616..62282e74ac 100644--- a/enterprise/server/utils/conversation_callback_utils.py+++ b/enterprise/server/utils/conversation_callback_utils.py@@ -1,6 +1,4 @@-import base64import json-import picklefrom datetime import datetimefrom server.logger import logger@@ -19,7 +17,6 @@ from openhands.core.schema.agent import AgentStatefrom openhands.events.event_store import EventStorefrom openhands.events.observation.agent import AgentStateChangedObservationfrom openhands.events.serialization.event import event_from_dict-from openhands.server.services.conversation_stats import ConversationStatsfrom openhands.storage import get_file_storefrom openhands.storage.files import FileStorefrom openhands.storage.locations import (@@ -280,16 +277,3 @@ def update_agent_state(user_id: str, conversation_id: str, content: bytes):)write_path = get_conversation_agent_state_filename(conversation_id, user_id)file_store.write(write_path, content)---def update_conversation_stats(user_id: str, conversation_id: str, content: bytes):- existing_convo_stats = ConversationStats(- file_store=file_store, conversation_id=conversation_id, user_id=user_id- )-- incoming_convo_stats = ConversationStats(None, conversation_id, None)- pickled = base64.b64decode(content)- incoming_convo_stats.restored_metrics = pickle.loads(pickled)-- # Merging automatically saves to file store- existing_convo_stats.merge_and_save(incoming_convo_stats)diff --git a/openhands/server/services/conversation_stats.py b/openhands/server/services/conversation_stats.pydeleted file mode 100644index f079739562..0000000000--- a/openhands/server/services/conversation_stats.py+++ /dev/null@@ -1,168 +0,0 @@-# IMPORTANT: LEGACY V0 CODE - Deprecated since version 1.0.0, scheduled for removal April 1, 2026-# This file is part of the legacy (V0) implementation of OpenHands and will be removed soon as we complete the migration to V1.-# OpenHands V1 uses the Software Agent SDK for the agentic core and runs a new application server. Please refer to:-# - V1 agentic core (SDK): https://github.com/OpenHands/software-agent-sdk-# - V1 application server (in this repo): openhands/app_server/-# Unless you are working on deprecation, please avoid extending this legacy file and consult the V1 codepaths above.-# Tag: Legacy-V0-# This module belongs to the old V0 web server. The V1 application server lives under openhands/app_server/.-import base64-import pickle-from threading import Lock--from openhands.core.logger import openhands_logger as logger-from openhands.llm.llm_registry import RegistryEvent-from openhands.llm.metrics import Metrics-from openhands.storage.files import FileStore-from openhands.storage.locations import (- get_conversation_stats_filename,-)---class ConversationStats:- def __init__(- self,- file_store: FileStore | None,- conversation_id: str,- user_id: str | None,- ):- self.metrics_path = get_conversation_stats_filename(conversation_id, user_id)- self.file_store = file_store- self.conversation_id = conversation_id- self.user_id = user_id-- self._save_lock = Lock()-- self.service_to_metrics: dict[str, Metrics] = {}- self.restored_metrics: dict[str, Metrics] = {}-- # Always attempt to restore registry if it exists- self.maybe_restore_metrics()-- def save_metrics(self):- if not self.file_store:- return-- with self._save_lock:- # Check for duplicate service IDs between restored and service metrics- duplicate_services = set(self.restored_metrics.keys()) & set(- self.service_to_metrics.keys()- )- if duplicate_services:- logger.error(- f'Duplicate service IDs found between restored and service metrics: {duplicate_services}. '- 'This should not happen as registered services should be removed from restored_metrics. '- 'Proceeding by preferring service_to_metrics values for duplicates.',- extra={- 'conversation_id': self.conversation_id,- 'duplicate_services': list(duplicate_services),- },- )-- # Combine both restored metrics and service metrics to avoid data loss- # Start with restored metrics (for services not yet registered)- combined_metrics = self.restored_metrics.copy()-- # Add service metrics (for registered services)- # Since we checked for duplicates above, this is safe- combined_metrics.update(self.service_to_metrics)-- pickled = pickle.dumps(combined_metrics)- serialized_metrics = base64.b64encode(pickled).decode('utf-8')- self.file_store.write(self.metrics_path, serialized_metrics)- logger.info(- 'Saved conversation stats',- extra={'conversation_id': self.conversation_id},- )-- def maybe_restore_metrics(self):- if not self.file_store or not self.conversation_id:- return-- try:- encoded = self.file_store.read(self.metrics_path)- pickled = base64.b64decode(encoded)- self.restored_metrics = pickle.loads(pickled)- logger.info(f'restored metrics: {self.conversation_id}')- except FileNotFoundError:- pass-- def get_combined_metrics(self) -> Metrics:- total_metrics = Metrics()- for metrics in self.service_to_metrics.values():- total_metrics.merge(metrics)- return total_metrics-- def get_metrics_for_service(self, service_id: str) -> Metrics:- if service_id not in self.service_to_metrics:- raise Exception(f'LLM service does not exist {service_id}')-- return self.service_to_metrics[service_id]-- def register_llm(self, event: RegistryEvent):- # Listen for llm creations and track their metrics- llm = event.llm- service_id = event.service_id-- if service_id in self.restored_metrics:- llm.metrics = self.restored_metrics[service_id].copy()- del self.restored_metrics[service_id]-- self.service_to_metrics[service_id] = llm.metrics-- def merge_and_save(self, conversation_stats: 'ConversationStats'):- """- Merge restored metrics from another ConversationStats into this one.-- Important:- - This method is intended to be used immediately after restoring metrics from- storage, before any LLM services are registered. In that state, only- `restored_metrics` should contain entries and `service_to_metrics` should- be empty. If either side has entries in `service_to_metrics`, we log an- error but continue execution.-- Behavior:- - Drop entries with zero accumulated_cost from both `restored_metrics` dicts- (self and incoming) before merging.- - Merge only `restored_metrics`. For duplicate keys, the incoming- `conversation_stats.restored_metrics` overwrites existing entries.- - Do NOT merge `service_to_metrics` here.- - Persist results by calling save_metrics().- """-- # If either side has active service metrics, log an error but proceed- if self.service_to_metrics or conversation_stats.service_to_metrics:- logger.error(- 'merge_and_save should be used only when service_to_metrics are empty; '- 'found active service metrics during merge. Proceeding anyway.',- extra={- 'conversation_id': self.conversation_id,- 'self_service_to_metrics_keys': list(- self.service_to_metrics.keys()- ),- 'incoming_service_to_metrics_keys': list(- conversation_stats.service_to_metrics.keys()- ),- },- )-- # Drop zero-cost entries from restored metrics only- def _drop_zero_cost(d: dict[str, Metrics]) -> None:- to_delete = [- k for k, v in d.items() if getattr(v, 'accumulated_cost', 0) == 0- ]- for k in to_delete:- del d[k]-- _drop_zero_cost(self.restored_metrics)- _drop_zero_cost(conversation_stats.restored_metrics)-- # Merge restored metrics, allowing incoming to overwrite- self.restored_metrics.update(conversation_stats.restored_metrics)-- # Save merged state- self.save_metrics()- logger.info(- 'Merged conversation stats',- extra={'conversation_id': self.conversation_id},- )diff --git a/openhands/storage/locations.py b/openhands/storage/locations.pyindex 7d68deeef7..5be22c83ab 100644--- a/openhands/storage/locations.py+++ b/openhands/storage/locations.py@@ -24,7 +24,3 @@ def get_conversation_metadata_filename(sid: str, user_id: str | None = None) ->def get_conversation_agent_state_filename(sid: str, user_id: str | None = None) -> str:return f'{get_conversation_dir(sid, user_id)}agent_state.pkl'---def get_conversation_stats_filename(sid: str, user_id: str | None = None) -> str:- return f'{get_conversation_dir(sid, user_id)}conversation_stats.pkl'diff --git a/openhands/utils/utils.py b/openhands/utils/utils.pyindex 923021fa99..5ed33e0361 100644--- a/openhands/utils/utils.py+++ b/openhands/utils/utils.py@@ -4,9 +4,6 @@ from copy import deepcopyfrom pydantic import SecretStrfrom openhands.core.config.openhands_config import OpenHandsConfig-from openhands.llm.llm_registry import LLMRegistry-from openhands.server.services.conversation_stats import ConversationStats-from openhands.storage import get_file_storefrom openhands.storage.data_models.settings import Settingsfrom openhands.utils.environment import get_effective_llm_base_url@@ -38,27 +35,3 @@ def setup_llm_config(config: OpenHandsConfig, settings: Settings) -> OpenHandsCo)config.set_llm_config(llm_config)return config---def create_registry_and_conversation_stats(- config: OpenHandsConfig,- sid: str,- user_id: str | None,- user_settings: Settings | None = None,-) -> tuple[LLMRegistry, ConversationStats, OpenHandsConfig]:- user_config = config- if user_settings:- user_config = setup_llm_config(config, user_settings)-- agent_cls = user_settings.agent_settings.agent if user_settings else None- llm_registry = LLMRegistry(user_config, agent_cls)- file_store = get_file_store(- file_store_type=config.file_store,- file_store_path=config.file_store_path,- file_store_web_hook_url=config.file_store_web_hook_url,- file_store_web_hook_headers=config.file_store_web_hook_headers,- file_store_web_hook_batch=config.file_store_web_hook_batch,- )- conversation_stats = ConversationStats(file_store, sid, user_id)- llm_registry.subscribe(conversation_stats.register_llm)- return llm_registry, conversation_stats, user_configdiff --git a/tests/unit/test_conversation_stats.py b/tests/unit/test_conversation_stats.pydeleted file mode 100644index ca6173087e..0000000000--- a/tests/unit/test_conversation_stats.py+++ /dev/null@@ -1,723 +0,0 @@-import base64-import pickle-from unittest.mock import patch--import pytest--from openhands.core.config import LLMConfig, OpenHandsConfig-from openhands.llm.llm import LLM-from openhands.llm.llm_registry import LLMRegistry, RegistryEvent-from openhands.llm.metrics import Metrics-from openhands.server.services.conversation_stats import ConversationStats-from openhands.storage.memory import InMemoryFileStore---def mock_file_store():- """Create a mock file store for testing."""- return InMemoryFileStore({})---def conversation_stats(mock_file_store):- """Create a ConversationStats instance for testing."""- return ConversationStats(- file_store=mock_file_store,- conversation_id='test-conversation-id',- user_id='test-user-id',- )---def mock_llm_registry():- """Create a mock LLM registry that properly simulates LLM registration."""- config = OpenHandsConfig()- registry = LLMRegistry(config=config, agent_cls=None, retry_listener=None)- return registry---def connected_registry_and_stats(mock_llm_registry, conversation_stats):- """Connect the LLMRegistry and ConversationStats properly."""- # Subscribe to LLM registry events to track metrics- mock_llm_registry.subscribe(conversation_stats.register_llm)- return mock_llm_registry, conversation_stats---def test_conversation_stats_initialization(conversation_stats):- """Test that ConversationStats initializes correctly."""- assert conversation_stats.conversation_id == 'test-conversation-id'- assert conversation_stats.user_id == 'test-user-id'- assert conversation_stats.service_to_metrics == {}- assert isinstance(conversation_stats.restored_metrics, dict)---def test_save_metrics(conversation_stats, mock_file_store):- """Test that metrics are saved correctly."""- # Add a service with metrics- service_id = 'test-service'- metrics = Metrics(model_name='gpt-4')- metrics.add_cost(0.05)- conversation_stats.service_to_metrics[service_id] = metrics-- # Save metrics- conversation_stats.save_metrics()-- # Verify that metrics were saved to the file store- try:- # Verify the saved content can be decoded and unpickled- encoded = mock_file_store.read(conversation_stats.metrics_path)- pickled = base64.b64decode(encoded)- restored = pickle.loads(pickled)-- assert service_id in restored- assert restored[service_id].accumulated_cost == 0.05- except FileNotFoundError:- pytest.fail(f'File not found: {conversation_stats.metrics_path}')---def test_maybe_restore_metrics(mock_file_store):- """Test that metrics are restored correctly."""- # Create metrics to save- service_id = 'test-service'- metrics = Metrics(model_name='gpt-4')- metrics.add_cost(0.1)- service_to_metrics = {service_id: metrics}-- # Serialize and save metrics- pickled = pickle.dumps(service_to_metrics)- serialized_metrics = base64.b64encode(pickled).decode('utf-8')-- # Create a new ConversationStats with pre-populated file store- conversation_id = 'test-conversation-id'- user_id = 'test-user-id'-- # Get the correct path using the same function as ConversationStats- from openhands.storage.locations import get_conversation_stats_filename-- metrics_path = get_conversation_stats_filename(conversation_id, user_id)-- # Write to the correct path- mock_file_store.write(metrics_path, serialized_metrics)-- # Create ConversationStats which should restore metrics- stats = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Verify metrics were restored- assert service_id in stats.restored_metrics- assert stats.restored_metrics[service_id].accumulated_cost == 0.1---def test_get_combined_metrics(conversation_stats):- """Test that combined metrics are calculated correctly."""- # Add multiple services with metrics- service1 = 'service1'- metrics1 = Metrics(model_name='gpt-4')- metrics1.add_cost(0.05)- metrics1.add_token_usage(- prompt_tokens=100,- completion_tokens=50,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=8000,- response_id='resp1',- )-- service2 = 'service2'- metrics2 = Metrics(model_name='gpt-3.5')- metrics2.add_cost(0.02)- metrics2.add_token_usage(- prompt_tokens=200,- completion_tokens=100,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=4000,- response_id='resp2',- )-- conversation_stats.service_to_metrics[service1] = metrics1- conversation_stats.service_to_metrics[service2] = metrics2-- # Get combined metrics- combined = conversation_stats.get_combined_metrics()-- # Verify combined metrics- assert combined.accumulated_cost == 0.07 # 0.05 + 0.02- assert combined.accumulated_token_usage.prompt_tokens == 300 # 100 + 200- assert combined.accumulated_token_usage.completion_tokens == 150 # 50 + 100- assert (- combined.accumulated_token_usage.context_window == 8000- ) # max of 8000 and 4000---def test_get_metrics_for_service(conversation_stats):- """Test that metrics for a specific service are retrieved correctly."""- # Add a service with metrics- service_id = 'test-service'- metrics = Metrics(model_name='gpt-4')- metrics.add_cost(0.05)- conversation_stats.service_to_metrics[service_id] = metrics-- # Get metrics for the service- retrieved_metrics = conversation_stats.get_metrics_for_service(service_id)-- # Verify metrics- assert retrieved_metrics.accumulated_cost == 0.05- assert retrieved_metrics is metrics # Should be the same object-- # Test getting metrics for non-existent service- # Use a specific exception message pattern instead of a blind Exception- with pytest.raises(Exception, match='LLM service does not exist'):- conversation_stats.get_metrics_for_service('non-existent-service')---def test_register_llm_with_new_service(conversation_stats):- """Test registering a new LLM service."""- # Create a real LLM instance with a mock config- llm_config = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Patch the LLM class to avoid actual API calls- with patch('openhands.llm.llm.litellm_completion'):- llm = LLM(service_id='new-service', config=llm_config)-- # Create a registry event- service_id = 'new-service'- event = RegistryEvent(llm=llm, service_id=service_id)-- # Register the LLM- conversation_stats.register_llm(event)-- # Verify the service was registered- assert service_id in conversation_stats.service_to_metrics- assert conversation_stats.service_to_metrics[service_id] is llm.metrics---def test_register_llm_with_restored_metrics(conversation_stats):- """Test registering an LLM service with restored metrics."""- # Create restored metrics- service_id = 'restored-service'- restored_metrics = Metrics(model_name='gpt-4')- restored_metrics.add_cost(0.1)- conversation_stats.restored_metrics = {service_id: restored_metrics}-- # Create a real LLM instance with a mock config- llm_config = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Patch the LLM class to avoid actual API calls- with patch('openhands.llm.llm.litellm_completion'):- llm = LLM(service_id=service_id, config=llm_config)-- # Create a registry event- event = RegistryEvent(llm=llm, service_id=service_id)-- # Register the LLM- conversation_stats.register_llm(event)-- # Verify the service was registered with restored metrics- assert service_id in conversation_stats.service_to_metrics- assert conversation_stats.service_to_metrics[service_id] is llm.metrics- assert llm.metrics.accumulated_cost == 0.1 # Restored cost-- # Verify the specific service was removed from restored_metrics- assert service_id not in conversation_stats.restored_metrics- assert hasattr(- conversation_stats, 'restored_metrics'- ) # The dict should still exist---def test_llm_registry_notifications(connected_registry_and_stats):- """Test that LLM registry notifications update conversation stats."""- mock_llm_registry, conversation_stats = connected_registry_and_stats-- # Create a new LLM through the registry- service_id = 'test-service'- llm_config = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Get LLM from registry (this should trigger the notification)- llm = mock_llm_registry.get_llm(service_id, llm_config)-- # Verify the service was registered in conversation stats- assert service_id in conversation_stats.service_to_metrics- assert conversation_stats.service_to_metrics[service_id] is llm.metrics-- # Add some metrics to the LLM- llm.metrics.add_cost(0.05)- llm.metrics.add_token_usage(- prompt_tokens=100,- completion_tokens=50,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=8000,- response_id='resp1',- )-- # Verify the metrics are reflected in conversation stats- assert conversation_stats.service_to_metrics[service_id].accumulated_cost == 0.05- assert (- conversation_stats.service_to_metrics[- service_id- ].accumulated_token_usage.prompt_tokens- == 100- )- assert (- conversation_stats.service_to_metrics[- service_id- ].accumulated_token_usage.completion_tokens- == 50- )-- # Get combined metrics and verify- combined = conversation_stats.get_combined_metrics()- assert combined.accumulated_cost == 0.05- assert combined.accumulated_token_usage.prompt_tokens == 100- assert combined.accumulated_token_usage.completion_tokens == 50---def test_multiple_llm_services(connected_registry_and_stats):- """Test tracking metrics for multiple LLM services."""- mock_llm_registry, conversation_stats = connected_registry_and_stats-- # Create multiple LLMs through the registry- service1 = 'service1'- service2 = 'service2'-- llm_config1 = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- llm_config2 = LLMConfig(- model='gpt-3.5-turbo',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Get LLMs from registry (this should trigger notifications)- llm1 = mock_llm_registry.get_llm(service1, llm_config1)- llm2 = mock_llm_registry.get_llm(service2, llm_config2)-- # Add different metrics to each LLM- llm1.metrics.add_cost(0.05)- llm1.metrics.add_token_usage(- prompt_tokens=100,- completion_tokens=50,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=8000,- response_id='resp1',- )-- llm2.metrics.add_cost(0.02)- llm2.metrics.add_token_usage(- prompt_tokens=200,- completion_tokens=100,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=4000,- response_id='resp2',- )-- # Verify services were registered in conversation stats- assert service1 in conversation_stats.service_to_metrics- assert service2 in conversation_stats.service_to_metrics-- # Verify individual metrics- assert conversation_stats.service_to_metrics[service1].accumulated_cost == 0.05- assert conversation_stats.service_to_metrics[service2].accumulated_cost == 0.02-- # Get combined metrics and verify- combined = conversation_stats.get_combined_metrics()- assert combined.accumulated_cost == 0.07 # 0.05 + 0.02- assert combined.accumulated_token_usage.prompt_tokens == 300 # 100 + 200- assert combined.accumulated_token_usage.completion_tokens == 150 # 50 + 100- assert (- combined.accumulated_token_usage.context_window == 8000- ) # max of 8000 and 4000---def test_register_llm_with_multiple_restored_services_bug(conversation_stats):- """Test that reproduces the bug where del self.restored_metrics deletes entire dict instead of specific service."""- # Create restored metrics for multiple services- service_id_1 = 'service-1'- service_id_2 = 'service-2'-- restored_metrics_1 = Metrics(model_name='gpt-4')- restored_metrics_1.add_cost(0.1)-- restored_metrics_2 = Metrics(model_name='gpt-3.5')- restored_metrics_2.add_cost(0.05)-- # Set up restored metrics for both services- conversation_stats.restored_metrics = {- service_id_1: restored_metrics_1,- service_id_2: restored_metrics_2,- }-- # Create LLM configs- llm_config_1 = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- llm_config_2 = LLMConfig(- model='gpt-3.5-turbo',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Patch the LLM class to avoid actual API calls- with patch('openhands.llm.llm.litellm_completion'):- # Register first LLM- llm_1 = LLM(service_id=service_id_1, config=llm_config_1)- event_1 = RegistryEvent(llm=llm_1, service_id=service_id_1)- conversation_stats.register_llm(event_1)-- # Verify first service was registered with restored metrics- assert service_id_1 in conversation_stats.service_to_metrics- assert llm_1.metrics.accumulated_cost == 0.1-- # After registering first service, restored_metrics should still contain service_id_2- assert service_id_2 in conversation_stats.restored_metrics-- # Register second LLM - this should also work with restored metrics- llm_2 = LLM(service_id=service_id_2, config=llm_config_2)- event_2 = RegistryEvent(llm=llm_2, service_id=service_id_2)- conversation_stats.register_llm(event_2)-- # Verify second service was registered with restored metrics- assert service_id_2 in conversation_stats.service_to_metrics- assert llm_2.metrics.accumulated_cost == 0.05-- # After both services are registered, restored_metrics should be empty- assert len(conversation_stats.restored_metrics) == 0---def test_save_and_restore_workflow(mock_file_store):- """Test the full workflow of saving and restoring metrics."""- # Create initial conversation stats- conversation_id = 'test-conversation-id'- user_id = 'test-user-id'-- stats1 = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Add a service with metrics- service_id = 'test-service'- metrics = Metrics(model_name='gpt-4')- metrics.add_cost(0.05)- metrics.add_token_usage(- prompt_tokens=100,- completion_tokens=50,- cache_read_tokens=0,- cache_write_tokens=0,- context_window=8000,- response_id='resp1',- )- stats1.service_to_metrics[service_id] = metrics-- # Save metrics- stats1.save_metrics()-- # Create a new conversation stats instance that should restore the metrics- stats2 = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Verify metrics were restored- assert service_id in stats2.restored_metrics- assert stats2.restored_metrics[service_id].accumulated_cost == 0.05- assert (- stats2.restored_metrics[service_id].accumulated_token_usage.prompt_tokens == 100- )- assert (- stats2.restored_metrics[service_id].accumulated_token_usage.completion_tokens- == 50- )-- # Create a real LLM instance with a mock config- llm_config = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- # Patch the LLM class to avoid actual API calls- with patch('openhands.llm.llm.litellm_completion'):- llm = LLM(service_id=service_id, config=llm_config)-- # Create a registry event- event = RegistryEvent(llm=llm, service_id=service_id)-- # Register the LLM to trigger restoration- stats2.register_llm(event)-- # Verify metrics were applied to the LLM- assert llm.metrics.accumulated_cost == 0.05- assert llm.metrics.accumulated_token_usage.prompt_tokens == 100- assert llm.metrics.accumulated_token_usage.completion_tokens == 50---def test_merge_conversation_stats_success_non_overlapping(mock_file_store):- """Merging two ConversationStats combines only restored metrics. Active metrics- (service_to_metrics) are not merged; if present, an error is logged but- execution continues. Incoming restored metrics overwrite duplicates.- """- stats_a = ConversationStats(- file_store=mock_file_store, conversation_id='conv-merge-a', user_id='user-x'- )- stats_b = ConversationStats(- file_store=mock_file_store, conversation_id='conv-merge-b', user_id='user-x'- )-- # Self active + restored- m_a_active = Metrics(model_name='model-a')- m_a_active.add_cost(0.1)- m_a_restored = Metrics(model_name='model-a')- m_a_restored.add_cost(0.2)- stats_a.service_to_metrics['a-active'] = m_a_active- stats_a.restored_metrics = {'a-restored': m_a_restored}-- # Other active + restored- m_b_active = Metrics(model_name='model-b')- m_b_active.add_cost(0.3)- m_b_restored = Metrics(model_name='model-b')- m_b_restored.add_cost(0.4)- stats_b.service_to_metrics['b-active'] = m_b_active- stats_b.restored_metrics = {'b-restored': m_b_restored}-- # Merge B into A- stats_a.merge_and_save(stats_b)-- # Active metrics should not be merged; only self's active metrics remain- assert set(stats_a.service_to_metrics.keys()) == {- 'a-active',- }-- # Restored metrics from both sides should be in A's restored_metrics- assert set(stats_a.restored_metrics.keys()) == {- 'a-restored',- 'b-restored',- }-- # The exact Metrics objects should be present (no copies)- assert stats_a.service_to_metrics['a-active'] is m_a_active- assert stats_a.restored_metrics['a-restored'] is m_a_restored- assert stats_a.restored_metrics['b-restored'] is m_b_restored-- # Merge triggers a save; confirm the saved blob decodes to expected keys- # The save_metrics method combines both service_to_metrics and restored_metrics- encoded = mock_file_store.read(stats_a.metrics_path)- pickled = base64.b64decode(encoded)- restored_dict = pickle.loads(pickled)- assert set(restored_dict.keys()) == {- 'a-active',- 'a-restored',- 'b-restored',- }--- 'self_side,other_side',- [- ('active', 'active'),- ('restored', 'active'),- ('active', 'restored'),- ('restored', 'restored'),- ],-)-def test_merge_conversation_stats_duplicates_overwrite_and_log_errors(- mock_file_store, self_side, other_side-):- stats_a = ConversationStats(- file_store=mock_file_store, conversation_id='conv-merge-a', user_id='user-x'- )- stats_b = ConversationStats(- file_store=mock_file_store, conversation_id='conv-merge-b', user_id='user-x'- )-- # Place the same service id on the specified sides- dupe_id = 'dupe-service'- m1 = Metrics(model_name='m')- m1.add_cost(0.1) # ensure not dropped- m2 = Metrics(model_name='m')- m2.add_cost(0.2) # ensure not dropped-- if self_side == 'active':- stats_a.service_to_metrics[dupe_id] = m1- else:- stats_a.restored_metrics[dupe_id] = m1-- if other_side == 'active':- stats_b.service_to_metrics[dupe_id] = m2- else:- stats_b.restored_metrics[dupe_id] = m2-- # Perform merge; should not raise and should log error internally if active metrics present- stats_a.merge_and_save(stats_b)-- # Only restored metrics are merged; duplicates are allowed with incoming overwriting- if other_side == 'restored':- assert dupe_id in stats_a.restored_metrics- assert stats_a.restored_metrics[dupe_id] is m2 # incoming overwrites existing- else:- # No restored metric came from incoming; existing restored stays as-is- if self_side == 'restored':- assert dupe_id in stats_a.restored_metrics- assert stats_a.restored_metrics[dupe_id] is m1- else:- assert dupe_id not in stats_a.restored_metrics---def test_save_metrics_preserves_restored_metrics_fix(mock_file_store):- """Test that save_metrics correctly preserves restored metrics for unregistered services."""- conversation_id = 'test-conversation-id'- user_id = 'test-user-id'-- # Step 1: Create initial conversation stats with multiple services- stats1 = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Add metrics for multiple services- service_a = 'service-a'- service_b = 'service-b'- service_c = 'service-c'-- metrics_a = Metrics(model_name='gpt-4')- metrics_a.add_cost(0.10)-- metrics_b = Metrics(model_name='gpt-3.5')- metrics_b.add_cost(0.05)-- metrics_c = Metrics(model_name='claude-3')- metrics_c.add_cost(0.08)-- stats1.service_to_metrics[service_a] = metrics_a- stats1.service_to_metrics[service_b] = metrics_b- stats1.service_to_metrics[service_c] = metrics_c-- # Save metrics (all three services should be saved)- stats1.save_metrics()-- # Step 2: Create new conversation stats instance (simulates app restart)- stats2 = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Verify all metrics were restored- assert service_a in stats2.restored_metrics- assert service_b in stats2.restored_metrics- assert service_c in stats2.restored_metrics- assert stats2.restored_metrics[service_a].accumulated_cost == 0.10- assert stats2.restored_metrics[service_b].accumulated_cost == 0.05- assert stats2.restored_metrics[service_c].accumulated_cost == 0.08-- # Step 3: Register only one LLM service (simulates partial LLM activation)- llm_config = LLMConfig(- model='gpt-4o',- api_key='test_key',- num_retries=2,- retry_min_wait=1,- retry_max_wait=2,- )-- with patch('openhands.llm.llm.litellm_completion'):- llm_a = LLM(service_id=service_a, config=llm_config)- event_a = RegistryEvent(llm=llm_a, service_id=service_a)- stats2.register_llm(event_a)-- # Verify service_a was moved from restored_metrics to service_to_metrics- assert service_a in stats2.service_to_metrics- assert service_a not in stats2.restored_metrics- assert stats2.service_to_metrics[service_a].accumulated_cost == 0.10-- # Verify services B and C are still in restored_metrics (not yet registered)- assert service_b in stats2.restored_metrics- assert service_c in stats2.restored_metrics- assert stats2.restored_metrics[service_b].accumulated_cost == 0.05- assert stats2.restored_metrics[service_c].accumulated_cost == 0.08-- # Step 4: Save metrics again (this is where the bug occurs)- stats2.save_metrics()-- # Step 5: Create a third conversation stats instance to verify what was saved- stats3 = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # FIXED: All services should be restored because save_metrics now combines both dictionaries- # Service A should be restored with its current metrics from service_to_metrics- assert service_a in stats3.restored_metrics- assert stats3.restored_metrics[service_a].accumulated_cost == 0.10-- # Services B and C should be preserved from restored_metrics- assert service_b in stats3.restored_metrics # FIXED: Now preserved- assert service_c in stats3.restored_metrics # FIXED: Now preserved- assert stats3.restored_metrics[service_b].accumulated_cost == 0.05- assert stats3.restored_metrics[service_c].accumulated_cost == 0.08---def test_save_metrics_throws_error_on_duplicate_service_ids(mock_file_store):- """Test updated: save_metrics should NOT raise on duplicate service IDs; it should prefer service_to_metrics and proceed."""- conversation_id = 'test-conversation-id'- user_id = 'test-user-id'-- stats = ConversationStats(- file_store=mock_file_store, conversation_id=conversation_id, user_id=user_id- )-- # Manually create a scenario with duplicate service IDs (this should never happen in normal operation)- service_id = 'duplicate-service'-- # Add to both restored_metrics and service_to_metrics- restored_metrics = Metrics(model_name='gpt-4')- restored_metrics.add_cost(0.10)- stats.restored_metrics[service_id] = restored_metrics-- service_metrics = Metrics(model_name='gpt-3.5')- service_metrics.add_cost(0.05)- stats.service_to_metrics[service_id] = service_metrics-- # Should not raise. Should save with service_to_metrics preferred.- stats.save_metrics()-- # Verify the saved content prefers service_to_metrics for duplicates- encoded = mock_file_store.read(stats.metrics_path)- pickled = base64.b64decode(encoded)- restored = pickle.loads(pickled)-- assert service_id in restored- assert restored[service_id].accumulated_cost == 0.05 # prefers service_to_metrics