π OpenHands: AI-Driven Development
Add detailed logging to remote runtime resume process (#8819)
Robert Brennan21af52560af7
diff --git a/openhands/runtime/impl/remote/remote_runtime.py b/openhands/runtime/impl/remote/remote_runtime.pyindex c3c20d58cc..0164118816 100644--- a/openhands/runtime/impl/remote/remote_runtime.py+++ b/openhands/runtime/impl/remote/remote_runtime.py@@ -100,7 +100,6 @@ class RemoteRuntime(ActionExecutionClient):self.available_hosts: dict[str, int] = {}def log(self, level: str, message: str, exc_info: bool | None = None) -> None:- message = f'[runtime session_id={self.sid} runtime_id={self.runtime_id or "unknown"}] {message}'getattr(logger, level)(message,stacklevel=2,@@ -128,14 +127,17 @@ class RemoteRuntime(ActionExecutionClient):self._runtime_initialized = Truedef _start_or_attach_to_runtime(self) -> None:+ self.log('info', 'Starting or attaching to runtime')existing_runtime = self._check_existing_runtime()if existing_runtime:- self.log('debug', f'Using existing runtime with ID: {self.runtime_id}')+ self.log('info', f'Using existing runtime with ID: {self.runtime_id}')elif self.attach_to_existing:+ self.log('info', f'Failed to find existing runtime for SID: {self.sid}')raise AgentRuntimeNotFoundError(f'Could not find existing runtime for SID: {self.sid}')else:+ self.log('info', 'No existing runtime found, starting a new one')self.send_status_message('STATUS$STARTING_CONTAINER')if self.config.sandbox.runtime_container_image is None:self.log(@@ -165,6 +167,7 @@ class RemoteRuntime(ActionExecutionClient):self.send_status_message(' ')def _check_existing_runtime(self) -> bool:+ self.log('info', f'Checking for existing runtime with session ID: {self.sid}')try:response = self._send_runtime_api_request('GET',@@ -172,12 +175,16 @@ class RemoteRuntime(ActionExecutionClient):)data = response.json()status = data.get('status')+ self.log('info', f'Found runtime with status: {status}')if status == 'running' or status == 'paused':self._parse_runtime_response(response)except httpx.HTTPError as e:if e.response.status_code == 404:+ self.log(+ 'info', f'No existing runtime found for session ID: {self.sid}'+ )return False- self.log('debug', f'Error while looking for remote runtime: {e}')+ self.log('error', f'Error while looking for remote runtime: {e}')raiseexcept json.decoder.JSONDecodeError as e:self.log(@@ -187,14 +194,25 @@ class RemoteRuntime(ActionExecutionClient):raiseif status == 'running':+ self.log('info', 'Found existing runtime in running state')return Trueelif status == 'stopped':- self.log('debug', 'Found existing remote runtime, but it is stopped')+ self.log('info', 'Found existing runtime, but it is stopped')return Falseelif status == 'paused':- self.log('debug', 'Found existing remote runtime, but it is paused')- self._resume_runtime()- return True+ self.log(+ 'info', 'Found existing runtime in paused state, attempting to resume'+ )+ try:+ self._resume_runtime()+ self.log('info', 'Successfully resumed paused runtime')+ return True+ except Exception as e:+ self.log(+ 'error', f'Failed to resume paused runtime: {e}', exc_info=True+ )+ # Return false to indicate we couldn't use the existing runtime+ return Falseelse:self.log('error', f'Invalid response from runtime API: {data}')return False@@ -287,16 +305,48 @@ class RemoteRuntime(ActionExecutionClient):3. Poll for the runtime to be ready4. Update env vars"""+ self.log('info', f'Attempting to resume runtime with ID: {self.runtime_id}')self.send_status_message('STATUS$STARTING_RUNTIME')- self._send_runtime_api_request(- 'POST',- f'{self.config.sandbox.remote_runtime_api_url}/resume',- json={'runtime_id': self.runtime_id},+ try:+ response = self._send_runtime_api_request(+ 'POST',+ f'{self.config.sandbox.remote_runtime_api_url}/resume',+ json={'runtime_id': self.runtime_id},+ )+ self.log(+ 'info',+ f'Resume API call successful with status code: {response.status_code}',+ )+ except Exception as e:+ self.log('error', f'Failed to call /resume API: {e}', exc_info=True)+ raise++ self.log(+ 'info', 'Runtime resume API call completed, waiting for it to be alive...')- self.log('info', 'Runtime resumed, waiting for it to be alive...')- self._wait_until_alive()- self.setup_initial_env()- self.log('info', 'Runtime resumed and alive.')+ try:+ self._wait_until_alive()+ self.log('info', 'Runtime is now alive after resume')+ except Exception as e:+ self.log(+ 'error',+ f'Runtime failed to become alive after resume: {e}',+ exc_info=True,+ )+ raise++ try:+ self.setup_initial_env()+ self.log('info', 'Successfully set up initial environment after resume')+ except Exception as e:+ self.log(+ 'error',+ f'Failed to set up initial environment after resume: {e}',+ exc_info=True,+ )+ raise++ self.log('info', 'Runtime successfully resumed and alive.')def _parse_runtime_response(self, response: httpx.Response) -> None:start_response = response.json()@@ -491,10 +541,32 @@ class RemoteRuntime(ActionExecutionClient):) from eelif hasattr(e, 'response') and e.response.status_code == 503:if self.config.sandbox.keep_runtime_alive:- self.log('warning', 'Runtime appears to be paused. Resuming...')- self._resume_runtime()- return super()._send_action_server_request(method, url, **kwargs)+ self.log(+ 'info',+ f'Runtime appears to be paused (503 response). Runtime ID: {self.runtime_id}, URL: {url}',+ )+ try:+ self._resume_runtime()+ self.log(+ 'info', 'Successfully resumed runtime after 503 response'+ )+ return super()._send_action_server_request(+ method, url, **kwargs+ )+ except Exception as resume_error:+ self.log(+ 'error',+ f'Failed to resume runtime after 503 response: {resume_error}',+ exc_info=True,+ )+ raise AgentRuntimeDisconnectedError(+ f'Runtime is paused and could not be resumed. Original error: {e}, Resume error: {resume_error}'+ ) from resume_errorelse:+ self.log(+ 'info',+ 'Runtime appears to be paused (503 response) but keep_runtime_alive is False',+ )raise AgentRuntimeDisconnectedError(f'Runtime is temporarily unavailable. This may be due to a restart or network issue, please try again. Original error: {e}') from e