π OpenHands: AI-Driven Development
Add Concurrency Limits to SandboxService (#11399)
Tim O'Farrell44578664edc9
diff --git a/openhands/app_server/sandbox/docker_sandbox_service.py b/openhands/app_server/sandbox/docker_sandbox_service.pyindex 0cdb81c10d..e31a6ac222 100644--- a/openhands/app_server/sandbox/docker_sandbox_service.py+++ b/openhands/app_server/sandbox/docker_sandbox_service.py@@ -74,6 +74,7 @@ class DockerSandboxService(SandboxService):exposed_ports: list[ExposedPort]health_check_path: str | Nonehttpx_client: httpx.AsyncClient+ max_num_sandboxes: intdocker_client: docker.DockerClient = field(default_factory=get_docker_client)def _find_unused_port(self) -> int:@@ -251,6 +252,9 @@ class DockerSandboxService(SandboxService):async def start_sandbox(self, sandbox_spec_id: str | None = None) -> SandboxInfo:"""Start a new sandbox."""+ # Enforce sandbox limits by cleaning up old sandboxes+ await self.pause_old_sandboxes(self.max_num_sandboxes - 1)+if sandbox_spec_id is None:sandbox_spec = await self.sandbox_spec_service.get_default_sandbox_spec()else:@@ -321,6 +325,9 @@ class DockerSandboxService(SandboxService):async def resume_sandbox(self, sandbox_id: str) -> bool:"""Resume a paused sandbox."""+ # Enforce sandbox limits by cleaning up old sandboxes+ await self.pause_old_sandboxes(self.max_num_sandboxes - 1)+try:if not sandbox_id.startswith(self.container_name_prefix):return False@@ -383,6 +390,10 @@ class DockerSandboxServiceInjector(SandboxServiceInjector):container_url_pattern: str = 'http://localhost:{port}'host_port: int = 3000container_name_prefix: str = 'oh-agent-server-'+ max_num_sandboxes: int = Field(+ default=5,+ description='Maximum number of sandboxes allowed to run simultaneously',+ )mounts: list[VolumeMount] = Field(default_factory=list)exposed_ports: list[ExposedPort] = Field(default_factory=lambda: [@@ -446,4 +457,5 @@ class DockerSandboxServiceInjector(SandboxServiceInjector):exposed_ports=self.exposed_ports,health_check_path=self.health_check_path,httpx_client=httpx_client,+ max_num_sandboxes=self.max_num_sandboxes,)diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex 9601e13b35..e3c69d6aa8 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -95,6 +95,7 @@ class RemoteSandboxService(SandboxService):resource_factor: intruntime_class: str | Nonestart_sandbox_timeout: int+ max_num_sandboxes: intuser_context: UserContexthttpx_client: httpx.AsyncClientdb_session: AsyncSession@@ -268,6 +269,9 @@ class RemoteSandboxService(SandboxService):async def start_sandbox(self, sandbox_spec_id: str | None = None) -> SandboxInfo:"""Start a new sandbox by creating a remote runtime."""try:+ # Enforce sandbox limits by cleaning up old sandboxes+ await self.pause_old_sandboxes(self.max_num_sandboxes - 1)+# Get sandbox specif sandbox_spec_id is None:sandbox_spec = (@@ -338,6 +342,9 @@ class RemoteSandboxService(SandboxService):async def resume_sandbox(self, sandbox_id: str) -> bool:"""Resume a paused sandbox."""+ # Enforce sandbox limits by cleaning up old sandboxes+ await self.pause_old_sandboxes(self.max_num_sandboxes - 1)+try:if not await self._get_stored_sandbox(sandbox_id):return False@@ -588,6 +595,10 @@ class RemoteSandboxServiceInjector(SandboxServiceInjector):'be in an error state.'),)+ max_num_sandboxes: int = Field(+ default=10,+ description='Maximum number of sandboxes allowed to run simultaneously',+ )async def inject(self, state: InjectorState, request: Request | None = None@@ -628,6 +639,7 @@ class RemoteSandboxServiceInjector(SandboxServiceInjector):resource_factor=self.resource_factor,runtime_class=self.runtime_class,start_sandbox_timeout=self.start_sandbox_timeout,+ max_num_sandboxes=self.max_num_sandboxes,user_context=user_context,httpx_client=httpx_client,db_session=db_session,diff --git a/openhands/app_server/sandbox/sandbox_service.py b/openhands/app_server/sandbox/sandbox_service.pyindex 526823b348..ad3d285309 100644--- a/openhands/app_server/sandbox/sandbox_service.py+++ b/openhands/app_server/sandbox/sandbox_service.py@@ -1,7 +1,11 @@import asynciofrom abc import ABC, abstractmethod-from openhands.app_server.sandbox.sandbox_models import SandboxInfo, SandboxPage+from openhands.app_server.sandbox.sandbox_models import (+ SandboxInfo,+ SandboxPage,+ SandboxStatus,+)from openhands.app_server.services.injector import Injectorfrom openhands.sdk.utils.models import DiscriminatedUnionMixin@@ -60,6 +64,61 @@ class SandboxService(ABC):Return False if the sandbox did not exist."""+ async def pause_old_sandboxes(self, max_num_sandboxes: int) -> list[str]:+ """Stop the oldest sandboxes if there are more than max_num_sandboxes running.++ Args:+ max_num_sandboxes: Maximum number of sandboxes to keep running++ Returns:+ List of sandbox IDs that were paused+ """+ if max_num_sandboxes <= 0:+ raise ValueError('max_num_sandboxes must be greater than 0')++ # Get all sandboxes (we'll search through all pages)+ all_sandboxes = []+ page_id = None++ while True:+ page = await self.search_sandboxes(page_id=page_id, limit=100)+ all_sandboxes.extend(page.items)++ if page.next_page_id is None:+ break+ page_id = page.next_page_id++ # Filter to only running sandboxes+ running_sandboxes = [+ sandbox+ for sandbox in all_sandboxes+ if sandbox.status == SandboxStatus.RUNNING+ ]++ # If we're within the limit, no cleanup needed+ if len(running_sandboxes) <= max_num_sandboxes:+ return []++ # Sort by creation time (oldest first)+ running_sandboxes.sort(key=lambda x: x.created_at)++ # Determine how many to pause+ num_to_pause = len(running_sandboxes) - max_num_sandboxes+ sandboxes_to_pause = running_sandboxes[:num_to_pause]++ # Stop the oldest sandboxes+ paused_sandbox_ids = []+ for sandbox in sandboxes_to_pause:+ try:+ success = await self.pause_sandbox(sandbox.id)+ if success:+ paused_sandbox_ids.append(sandbox.id)+ except Exception:+ # Continue trying to pause other sandboxes even if one fails+ pass++ return paused_sandbox_ids+class SandboxServiceInjector(DiscriminatedUnionMixin, Injector[SandboxService], ABC):passdiff --git a/tests/unit/app_server/test_docker_sandbox_service.py b/tests/unit/app_server/test_docker_sandbox_service.pyindex 211ff023b2..b3fc2fcd41 100644--- a/tests/unit/app_server/test_docker_sandbox_service.py+++ b/tests/unit/app_server/test_docker_sandbox_service.py@@ -80,6 +80,7 @@ def service(mock_sandbox_spec_service, mock_httpx_client, mock_docker_client):],health_check_path='/health',httpx_client=mock_httpx_client,+ max_num_sandboxes=3,docker_client=mock_docker_client,)@@ -354,7 +355,12 @@ class TestDockerSandboxService:service.docker_client.containers.run.return_value = mock_container- with patch.object(service, '_find_unused_port', side_effect=[12345, 12346]):+ with (+ patch.object(service, '_find_unused_port', side_effect=[12345, 12346]),+ patch.object(+ service, 'pause_old_sandboxes', return_value=[]+ ) as mock_cleanup,+ ):# Executeresult = await service.start_sandbox()@@ -362,6 +368,9 @@ class TestDockerSandboxService:assert result is not Noneassert result.id == 'oh-test-test_container_id'+ # Verify cleanup was called with the correct limit+ mock_cleanup.assert_called_once_with(2)+# Verify container was created with correct parametersservice.docker_client.containers.run.assert_called_once()call_args = service.docker_client.containers.run.call_args@@ -395,7 +404,10 @@ class TestDockerSandboxService:}service.docker_client.containers.run.return_value = mock_container- with patch.object(service, '_find_unused_port', return_value=12345):+ with (+ patch.object(service, '_find_unused_port', return_value=12345),+ patch.object(service, 'pause_old_sandboxes', return_value=[]),+ ):# Executeawait service.start_sandbox(sandbox_spec_id='custom-spec')@@ -412,7 +424,10 @@ class TestDockerSandboxService:mock_sandbox_spec_service.get_sandbox_spec.return_value = None# Execute & Verify- with pytest.raises(ValueError, match='Sandbox Spec not found'):+ with (+ patch.object(service, 'pause_old_sandboxes', return_value=[]),+ pytest.raises(ValueError, match='Sandbox Spec not found'),+ ):await service.start_sandbox(sandbox_spec_id='nonexistent')async def test_start_sandbox_docker_error(self, service):@@ -422,10 +437,12 @@ class TestDockerSandboxService:'Failed to create container')- with patch.object(service, '_find_unused_port', return_value=12345):- # Execute & Verify- with pytest.raises(SandboxError, match='Failed to start container'):- await service.start_sandbox()+ with (+ patch.object(service, '_find_unused_port', return_value=12345),+ patch.object(service, 'pause_old_sandboxes', return_value=[]),+ pytest.raises(SandboxError, match='Failed to start container'),+ ):+ await service.start_sandbox()async def test_resume_sandbox_from_paused(self, service):"""Test resuming a paused sandbox."""@@ -434,13 +451,18 @@ class TestDockerSandboxService:mock_container.status = 'paused'service.docker_client.containers.get.return_value = mock_container- # Execute- result = await service.resume_sandbox('oh-test-abc123')+ with patch.object(+ service, 'pause_old_sandboxes', return_value=[]+ ) as mock_cleanup:+ # Execute+ result = await service.resume_sandbox('oh-test-abc123')# Verifyassert result is Truemock_container.unpause.assert_called_once()mock_container.start.assert_not_called()+ # Verify cleanup was called with the correct limit+ mock_cleanup.assert_called_once_with(2)async def test_resume_sandbox_from_exited(self, service):"""Test resuming an exited sandbox."""@@ -449,22 +471,32 @@ class TestDockerSandboxService:mock_container.status = 'exited'service.docker_client.containers.get.return_value = mock_container- # Execute- result = await service.resume_sandbox('oh-test-abc123')+ with patch.object(+ service, 'pause_old_sandboxes', return_value=[]+ ) as mock_cleanup:+ # Execute+ result = await service.resume_sandbox('oh-test-abc123')# Verifyassert result is Truemock_container.start.assert_called_once()mock_container.unpause.assert_not_called()+ # Verify cleanup was called with the correct limit+ mock_cleanup.assert_called_once_with(2)async def test_resume_sandbox_wrong_prefix(self, service):"""Test resuming sandbox with wrong prefix."""- # Execute- result = await service.resume_sandbox('wrong-prefix-abc123')+ with patch.object(+ service, 'pause_old_sandboxes', return_value=[]+ ) as mock_cleanup:+ # Execute+ result = await service.resume_sandbox('wrong-prefix-abc123')# Verifyassert result is Falseservice.docker_client.containers.get.assert_not_called()+ # Verify cleanup was still called+ mock_cleanup.assert_called_once_with(2)async def test_resume_sandbox_not_found(self, service):"""Test resuming non-existent sandbox."""@@ -473,11 +505,16 @@ class TestDockerSandboxService:'Container not found')- # Execute- result = await service.resume_sandbox('oh-test-abc123')+ with patch.object(+ service, 'pause_old_sandboxes', return_value=[]+ ) as mock_cleanup:+ # Execute+ result = await service.resume_sandbox('oh-test-abc123')# Verifyassert result is False+ # Verify cleanup was still called+ mock_cleanup.assert_called_once_with(2)async def test_pause_sandbox_success(self, service):"""Test pausing a running sandbox."""diff --git a/tests/unit/app_server/test_sandbox_service.py b/tests/unit/app_server/test_sandbox_service.pynew file mode 100644index 0000000000..9a65131821--- /dev/null+++ b/tests/unit/app_server/test_sandbox_service.py@@ -0,0 +1,353 @@+"""Tests for SandboxService base class.++This module tests the SandboxService base class implementation, focusing on:+- pause_old_sandboxes method functionality+- Proper handling of pagination when searching sandboxes+- Correct filtering of running vs non-running sandboxes+- Proper sorting by creation time (oldest first)+- Error handling and edge cases+"""++from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest++from openhands.app_server.sandbox.sandbox_models import (+ SandboxInfo,+ SandboxPage,+ SandboxStatus,+)+from openhands.app_server.sandbox.sandbox_service import SandboxService+++class MockSandboxService(SandboxService):+ """Mock implementation of SandboxService for testing."""++ def __init__(self):+ self.search_sandboxes_mock = AsyncMock()+ self.get_sandbox_mock = AsyncMock()+ self.start_sandbox_mock = AsyncMock()+ self.resume_sandbox_mock = AsyncMock()+ self.pause_sandbox_mock = AsyncMock()+ self.delete_sandbox_mock = AsyncMock()++ async def search_sandboxes(+ self, page_id: str | None = None, limit: int = 100+ ) -> SandboxPage:+ return await self.search_sandboxes_mock(page_id=page_id, limit=limit)++ async def get_sandbox(self, sandbox_id: str) -> SandboxInfo | None:+ return await self.get_sandbox_mock(sandbox_id)++ async def start_sandbox(self, sandbox_spec_id: str | None = None) -> SandboxInfo:+ return await self.start_sandbox_mock(sandbox_spec_id)++ async def resume_sandbox(self, sandbox_id: str) -> bool:+ return await self.resume_sandbox_mock(sandbox_id)++ async def pause_sandbox(self, sandbox_id: str) -> bool:+ return await self.pause_sandbox_mock(sandbox_id)++ async def delete_sandbox(self, sandbox_id: str) -> bool:+ return await self.delete_sandbox_mock(sandbox_id)+++def create_sandbox_info(+ sandbox_id: str,+ status: SandboxStatus,+ created_at: datetime,+ created_by_user_id: str | None = None,+ sandbox_spec_id: str = 'test-spec',+) -> SandboxInfo:+ """Helper function to create SandboxInfo objects for testing."""+ return SandboxInfo(+ id=sandbox_id,+ created_by_user_id=created_by_user_id,+ sandbox_spec_id=sandbox_spec_id,+ status=status,+ session_api_key='test-api-key' if status == SandboxStatus.RUNNING else None,+ created_at=created_at,+ )+++def mock_sandbox_service():+ """Fixture providing a mock sandbox service."""+ return MockSandboxService()+++class TestCleanupOldSandboxes:+ """Test cases for the pause_old_sandboxes method."""++ @pytest.mark.asyncio+ async def test_cleanup_with_no_sandboxes(self, mock_sandbox_service):+ """Test cleanup when there are no sandboxes."""+ # Setup: No sandboxes+ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=[], next_page_id=None+ )++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=5)++ # Verify+ assert result == []+ mock_sandbox_service.search_sandboxes_mock.assert_called_once_with(+ page_id=None, limit=100+ )+ mock_sandbox_service.pause_sandbox_mock.assert_not_called()++ @pytest.mark.asyncio+ async def test_cleanup_within_limit(self, mock_sandbox_service):+ """Test cleanup when sandbox count is within the limit."""+ # Setup: 3 running sandboxes, limit is 5+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info('sb2', SandboxStatus.RUNNING, now - timedelta(hours=2)),+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=5)++ # Verify+ assert result == []+ mock_sandbox_service.pause_sandbox_mock.assert_not_called()++ @pytest.mark.asyncio+ async def test_cleanup_exceeds_limit(self, mock_sandbox_service):+ """Test cleanup when sandbox count exceeds the limit."""+ # Setup: 5 running sandboxes, limit is 3+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info(+ 'sb1', SandboxStatus.RUNNING, now - timedelta(hours=5)+ ), # oldest+ create_sandbox_info(+ 'sb2', SandboxStatus.RUNNING, now - timedelta(hours=4)+ ), # second oldest+ create_sandbox_info(+ 'sb3', SandboxStatus.RUNNING, now - timedelta(hours=3)+ ), # should be stopped+ create_sandbox_info(+ 'sb4', SandboxStatus.RUNNING, now - timedelta(hours=2)+ ), # should remain+ create_sandbox_info(+ 'sb5', SandboxStatus.RUNNING, now - timedelta(hours=1)+ ), # newest+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )+ mock_sandbox_service.pause_sandbox_mock.return_value = True++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=2)++ # Verify: Should pause the 2 oldest sandboxes+ assert result == ['sb1', 'sb2', 'sb3']+ assert mock_sandbox_service.pause_sandbox_mock.call_count == 3++ @pytest.mark.asyncio+ async def test_cleanup_filters_non_running_sandboxes(self, mock_sandbox_service):+ """Test that cleanup only considers running sandboxes."""+ # Setup: Mix of running and non-running sandboxes+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=5)),+ create_sandbox_info(+ 'sb2', SandboxStatus.PAUSED, now - timedelta(hours=4)+ ), # should be ignored+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info(+ 'sb4', SandboxStatus.ERROR, now - timedelta(hours=2)+ ), # should be ignored+ create_sandbox_info('sb5', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )+ mock_sandbox_service.pause_sandbox_mock.return_value = True++ # Execute: Limit is 2, but only 3 are running+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=2)++ # Verify: Should stop only 1 sandbox (the oldest running one)+ assert len(result) == 1+ assert 'sb1' in result+ mock_sandbox_service.pause_sandbox_mock.assert_called_once_with('sb1')++ @pytest.mark.asyncio+ async def test_cleanup_with_pagination(self, mock_sandbox_service):+ """Test cleanup handles pagination correctly."""+ # Setup: Multiple pages of sandboxes+ now = datetime.now(timezone.utc)++ # First page+ page1_sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info('sb2', SandboxStatus.RUNNING, now - timedelta(hours=2)),+ ]++ # Second page+ page2_sandboxes = [+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ def search_side_effect(page_id=None, limit=100):+ if page_id is None:+ return SandboxPage(items=page1_sandboxes, next_page_id='page2')+ elif page_id == 'page2':+ return SandboxPage(items=page2_sandboxes, next_page_id=None)++ mock_sandbox_service.search_sandboxes_mock.side_effect = search_side_effect+ mock_sandbox_service.pause_sandbox_mock.return_value = True++ # Execute: Limit is 2, total is 3+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=2)++ # Verify: Should stop the oldest sandbox+ assert len(result) == 1+ assert 'sb1' in result+ assert mock_sandbox_service.search_sandboxes_mock.call_count == 2++ @pytest.mark.asyncio+ async def test_cleanup_handles_pause_failures(self, mock_sandbox_service):+ """Test cleanup continues when some pause operations fail."""+ # Setup: 4 running sandboxes, limit is 2+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=4)),+ create_sandbox_info('sb2', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=2)),+ create_sandbox_info('sb4', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )++ # Setup: First pause fails, second succeeds+ def pause_side_effect(sandbox_id):+ if sandbox_id == 'sb1':+ return False # Simulate failure+ return True++ mock_sandbox_service.pause_sandbox_mock.side_effect = pause_side_effect++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=2)++ # Verify: Should only include successfully paused sandbox+ assert len(result) == 1+ assert 'sb2' in result+ assert mock_sandbox_service.pause_sandbox_mock.call_count == 2++ @pytest.mark.asyncio+ async def test_cleanup_handles_pause_exceptions(self, mock_sandbox_service):+ """Test cleanup continues when pause operations raise exceptions."""+ # Setup: 3 running sandboxes, limit is 1+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info('sb2', SandboxStatus.RUNNING, now - timedelta(hours=2)),+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )++ # Setup: First pause raises exception, second succeeds+ def pause_side_effect(sandbox_id):+ if sandbox_id == 'sb1':+ raise Exception('Delete failed')+ return True++ mock_sandbox_service.pause_sandbox_mock.side_effect = pause_side_effect++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=1)++ # Verify: Should only include successfully paused sandbox+ assert len(result) == 1+ assert 'sb2' in result+ assert mock_sandbox_service.pause_sandbox_mock.call_count == 2++ @pytest.mark.asyncio+ async def test_cleanup_invalid_max_num_sandboxes(self, mock_sandbox_service):+ """Test cleanup raises ValueError for invalid max_num_sandboxes."""+ # Test zero+ with pytest.raises(+ ValueError, match='max_num_sandboxes must be greater than 0'+ ):+ await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=0)++ # Test negative+ with pytest.raises(+ ValueError, match='max_num_sandboxes must be greater than 0'+ ):+ await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=-1)++ @pytest.mark.asyncio+ async def test_cleanup_sorts_by_creation_time(self, mock_sandbox_service):+ """Test that cleanup properly sorts sandboxes by creation time."""+ # Setup: Sandboxes in random order by creation time+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb_newest', SandboxStatus.RUNNING, now), # newest+ create_sandbox_info(+ 'sb_oldest', SandboxStatus.RUNNING, now - timedelta(hours=5)+ ), # oldest+ create_sandbox_info(+ 'sb_middle', SandboxStatus.RUNNING, now - timedelta(hours=2)+ ), # middle+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )+ mock_sandbox_service.pause_sandbox_mock.return_value = True++ # Execute: Keep only 1 sandbox+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=1)++ # Verify: Should stop the 2 oldest sandboxes in order+ assert len(result) == 2+ assert 'sb_oldest' in result+ assert 'sb_middle' in result++ # Verify pause was called in the correct order (oldest first)+ calls = mock_sandbox_service.pause_sandbox_mock.call_args_list+ assert calls[0][0][0] == 'sb_oldest'+ assert calls[1][0][0] == 'sb_middle'++ @pytest.mark.asyncio+ async def test_cleanup_exact_limit(self, mock_sandbox_service):+ """Test cleanup when sandbox count exactly equals the limit."""+ # Setup: Exactly 3 running sandboxes, limit is 3+ now = datetime.now(timezone.utc)+ sandboxes = [+ create_sandbox_info('sb1', SandboxStatus.RUNNING, now - timedelta(hours=3)),+ create_sandbox_info('sb2', SandboxStatus.RUNNING, now - timedelta(hours=2)),+ create_sandbox_info('sb3', SandboxStatus.RUNNING, now - timedelta(hours=1)),+ ]++ mock_sandbox_service.search_sandboxes_mock.return_value = SandboxPage(+ items=sandboxes, next_page_id=None+ )++ # Execute+ result = await mock_sandbox_service.pause_old_sandboxes(max_num_sandboxes=3)++ # Verify: No sandboxes should be stopped+ assert result == []+ mock_sandbox_service.pause_sandbox_mock.assert_not_called()