π OpenHands: AI-Driven Development
Remove deprecated webhook file stores (V0 cleanup) (#14164)
Tim O'Farrell4c8179cd0879
diff --git a/openhands/app_server/secrets/file_secrets_store.py b/openhands/app_server/secrets/file_secrets_store.pyindex 3c2093cecc..1cc189ff01 100644--- a/openhands/app_server/secrets/file_secrets_store.py+++ b/openhands/app_server/secrets/file_secrets_store.py@@ -42,8 +42,5 @@ class FileSecretsStore(SecretsStore):file_store = get_file_store(file_store_type=config.file_store,file_store_path=config.file_store_path,- file_store_web_hook_url=config.file_store_web_hook_url,- file_store_web_hook_headers=config.file_store_web_hook_headers,- file_store_web_hook_batch=config.file_store_web_hook_batch,)return FileSecretsStore(file_store)diff --git a/openhands/app_server/settings/file_settings_store.py b/openhands/app_server/settings/file_settings_store.pyindex 361c91becc..f22f19013e 100644--- a/openhands/app_server/settings/file_settings_store.py+++ b/openhands/app_server/settings/file_settings_store.py@@ -43,8 +43,5 @@ class FileSettingsStore(SettingsStore):file_store = get_file_store(file_store_type=config.file_store,file_store_path=config.file_store_path,- file_store_web_hook_url=config.file_store_web_hook_url,- file_store_web_hook_headers=config.file_store_web_hook_headers,- file_store_web_hook_batch=config.file_store_web_hook_batch,)return FileSettingsStore(file_store)diff --git a/openhands/core/config/openhands_config.py b/openhands/core/config/openhands_config.pyindex 4b0f32c117..3c91a38baf 100644--- a/openhands/core/config/openhands_config.py+++ b/openhands/core/config/openhands_config.py@@ -40,8 +40,6 @@ class OpenHandsConfig(BaseModel):runtime: Runtime environment identifier.file_store: Type of file store to use.file_store_path: Path to the file store.- file_store_web_hook_url: Optional url for file store web hook- file_store_web_hook_headers: Optional headers for file_store web hookenable_browser: Whether to enable the browser environmentsave_trajectory_path: Either a folder path to store trajectories with auto-generated filenames, or a designated trajectory file path.save_screenshots_in_trajectory: Whether to save screenshots in trajectory (in encoded image format).@@ -77,9 +75,6 @@ class OpenHandsConfig(BaseModel):runtime: str = Field(default='docker')file_store: str = Field(default='local')file_store_path: str = Field(default='~/.openhands')- file_store_web_hook_url: str | None = Field(default=None)- file_store_web_hook_headers: dict | None = Field(default=None)- file_store_web_hook_batch: bool = Field(default=False)enable_browser: bool = Field(default=True)save_trajectory_path: str | None = Field(default=None)save_screenshots_in_trajectory: bool = Field(default=False)diff --git a/openhands/server/shared.py b/openhands/server/shared.pyindex a5e0e44eb8..815d018a82 100644--- a/openhands/server/shared.py+++ b/openhands/server/shared.py@@ -32,9 +32,6 @@ server_config: ServerConfig = server_config_interfacefile_store: FileStore = get_file_store(file_store_type=config.file_store,file_store_path=config.file_store_path,- file_store_web_hook_url=config.file_store_web_hook_url,- file_store_web_hook_headers=config.file_store_web_hook_headers,- file_store_web_hook_batch=config.file_store_web_hook_batch,)client_manager = Nonediff --git a/openhands/storage/__init__.py b/openhands/storage/__init__.pyindex 8ac6f47a9e..0ecae1be06 100644--- a/openhands/storage/__init__.py+++ b/openhands/storage/__init__.py@@ -1,61 +1,21 @@-import os--import httpx--from openhands.storage.batched_web_hook import BatchedWebHookFileStorefrom openhands.storage.files import FileStorefrom openhands.storage.google_cloud import GoogleCloudFileStorefrom openhands.storage.local import LocalFileStorefrom openhands.storage.memory import InMemoryFileStorefrom openhands.storage.s3 import S3FileStore-from openhands.storage.web_hook import WebHookFileStore-from openhands.utils.http_session import httpx_verify_optiondef get_file_store(file_store_type: str,file_store_path: str | None = None,- file_store_web_hook_url: str | None = None,- file_store_web_hook_headers: dict | None = None,- file_store_web_hook_batch: bool = False,) -> FileStore:- store: FileStoreif file_store_type == 'local':if file_store_path is None:raise ValueError('file_store_path is required for local file store')- store = LocalFileStore(file_store_path)+ return LocalFileStore(file_store_path)elif file_store_type == 's3':- store = S3FileStore(file_store_path)+ return S3FileStore(file_store_path)elif file_store_type == 'google_cloud':- store = GoogleCloudFileStore(file_store_path)+ return GoogleCloudFileStore(file_store_path)else:- store = InMemoryFileStore()- if file_store_web_hook_url:- if file_store_web_hook_headers is None:- # Fallback to default headers. Use the session api key if it is defined in the env.- file_store_web_hook_headers = {}- if os.getenv('SESSION_API_KEY'):- file_store_web_hook_headers['X-Session-API-Key'] = os.getenv(- 'SESSION_API_KEY'- )-- client = httpx.Client(- headers=file_store_web_hook_headers or {},- verify=httpx_verify_option(),- )-- if file_store_web_hook_batch:- # Use batched webhook file store- store = BatchedWebHookFileStore(- store,- file_store_web_hook_url,- client,- )- else:- # Use regular webhook file store- store = WebHookFileStore(- store,- file_store_web_hook_url,- client,- )- return store+ return InMemoryFileStore()diff --git a/openhands/storage/batched_web_hook.py b/openhands/storage/batched_web_hook.pydeleted file mode 100644index 7cd6d17fc4..0000000000--- a/openhands/storage/batched_web_hook.py+++ /dev/null@@ -1,268 +0,0 @@-import threading-from typing import Optional--import httpx-import tenacity--from openhands.core.logger import openhands_logger as logger-from openhands.storage.files import FileStore-from openhands.utils.async_utils import EXECUTOR-from openhands.utils.http_session import httpx_verify_option--# Constants for batching configuration-WEBHOOK_BATCH_TIMEOUT_SECONDS = 5.0-WEBHOOK_BATCH_SIZE_LIMIT_BYTES = 1048576 # 1MB---class BatchedWebHookFileStore(FileStore):- """File store which batches updates before sending them to a webhook.-- This class wraps another FileStore implementation and sends HTTP requests- to a specified URL when files are written or deleted. Updates are batched- and sent together after a certain amount of time or size exceeds a threshold.- Time is counted from the last insert.-- Attributes:- file_store: The underlying FileStore implementation- base_url: The base URL for webhook requests- client: The HTTP client used to make webhook requests- batch_timeout_seconds: Time in seconds after which a batch is sent (default: WEBHOOK_BATCH_TIMEOUT_SECONDS)- batch_size_limit_bytes: Size limit in bytes after which a batch is sent (default: WEBHOOK_BATCH_SIZE_LIMIT_BYTES)- _batch_lock: Lock for thread-safe access to the batch- _batch: Dictionary of pending file updates- _batch_timer: Timer for sending batches after timeout- _batch_size: Current size of the batch in bytes- """-- file_store: FileStore- base_url: str- client: httpx.Client- batch_timeout_seconds: float- batch_size_limit_bytes: int- _batch_lock: threading.Lock- _batch: dict[str, tuple[str, str | bytes | None]]- _batch_timer: Optional[threading.Timer]- _batch_size: int-- def __init__(- self,- file_store: FileStore,- base_url: str,- client: Optional[httpx.Client] = None,- batch_timeout_seconds: Optional[float] = None,- batch_size_limit_bytes: Optional[int] = None,- ):- """Initialize a BatchedWebHookFileStore.-- Args:- file_store: The underlying FileStore implementation- base_url: The base URL for webhook requests- client: Optional HTTP client to use for requests. If None, a new client will be created.- batch_timeout_seconds: Time in seconds after which a batch is sent.- If None, uses the default constant WEBHOOK_BATCH_TIMEOUT_SECONDS.- batch_size_limit_bytes: Size limit in bytes after which a batch is sent.- If None, uses the default constant WEBHOOK_BATCH_SIZE_LIMIT_BYTES.- """- self.file_store = file_store- self.base_url = base_url- if client is None:- client = httpx.Client(verify=httpx_verify_option())- self.client = client-- # Use provided values or default constants- self.batch_timeout_seconds = (- batch_timeout_seconds or WEBHOOK_BATCH_TIMEOUT_SECONDS- )- self.batch_size_limit_bytes = (- batch_size_limit_bytes or WEBHOOK_BATCH_SIZE_LIMIT_BYTES- )-- # Initialize batch state- self._batch_lock = threading.Lock()- self._batch = {} # Maps path -> (operation, content)- self._batch_timer = None- self._batch_size = 0-- def write(self, path: str, contents: str | bytes) -> None:- """Write contents to a file and queue a webhook update.-- Args:- path: The path to write to- contents: The contents to write- """- self.file_store.write(path, contents)- self._queue_update(path, 'write', contents)-- def read(self, path: str) -> str:- """Read contents from a file.-- Args:- path: The path to read from-- Returns:- The contents of the file- """- return self.file_store.read(path)-- def list(self, path: str) -> list[str]:- """List files in a directory.-- Args:- path: The directory path to list-- Returns:- A list of file paths- """- return self.file_store.list(path)-- def delete(self, path: str) -> None:- """Delete a file and queue a webhook update.-- Args:- path: The path to delete- """- self.file_store.delete(path)- self._queue_update(path, 'delete', None)-- def _queue_update(- self, path: str, operation: str, contents: str | bytes | None- ) -> None:- """Queue an update to be sent to the webhook.-- Args:- path: The path that was modified- operation: The operation performed ("write" or "delete")- contents: The contents that were written (None for delete operations)- """- with self._batch_lock:- # Calculate content size- content_size = 0- if contents is not None:- if isinstance(contents, str):- content_size = len(contents.encode('utf-8'))- else:- content_size = len(contents)-- # Update batch size calculation- # If this path already exists in the batch, subtract its previous size- if path in self._batch:- prev_op, prev_contents = self._batch[path]- if prev_contents is not None:- if isinstance(prev_contents, str):- self._batch_size -= len(prev_contents.encode('utf-8'))- else:- self._batch_size -= len(prev_contents)-- # Add new content size- self._batch_size += content_size-- # Add to batch- self._batch[path] = (operation, contents)-- # Check if we need to send the batch due to size limit- if self._batch_size >= self.batch_size_limit_bytes:- self._enqueue_batch_from_lock()- return-- # Start or reset the timer for sending the batch- if self._batch_timer is not None:- self._batch_timer.cancel()- self._batch_timer = None-- timer = threading.Timer(- self.batch_timeout_seconds, self._enqueue_batch_from_timer- )- timer.daemon = True- timer.start()- self._batch_timer = timer-- def _enqueue_batch_from_timer(self) -> None:- """Send the batch from the timer thread.- This method is called by the timer and submits the actual sending to the executor.- """- with self._batch_lock:- self._enqueue_batch_from_lock()-- def _enqueue_batch_from_lock(self, background=True) -> None:- """- Must have lock before calling. Will reset the batch state and send the current one.- Uses executor by default, but can perform synchronously by setting background=False- """- batch_to_send = self._batch- self._batch = {}- self._batch_size = 0- if background:- EXECUTOR.submit(self._send_batch, batch_to_send)- else:- self._send_batch(batch_to_send)- # Cancel any pending timer- if self._batch_timer is not None:- self._batch_timer.cancel()- self._batch_timer = None-- def _send_batch(- self, batch_to_send: dict[str, tuple[str, str | bytes | None]]- ) -> None:- """Send the current batch of updates to the webhook as a single request.- This method acquires the batch lock and processes all pending updates in one batch.- """- # Process the entire batch in a single request- if batch_to_send:- try:- self._send_batch_request(batch_to_send)- except Exception:- logger.exception('Error sending webhook batch')-- @tenacity.retry(- wait=tenacity.wait_fixed(1),- stop=tenacity.stop_after_attempt(3),- )- def _send_batch_request(- self, batch: dict[str, tuple[str, str | bytes | None]]- ) -> None:- """Send a single batch request to the webhook URL with all updates.-- This method is retried up to 3 times with a 1-second delay between attempts.-- Args:- batch: Dictionary mapping paths to (operation, contents) tuples-- Raises:- httpx.HTTPStatusError: If the webhook request fails- """- # Prepare the batch payload- batch_payload = []-- for path, (operation, contents) in batch.items():- item = {- 'method': 'POST' if operation == 'write' else 'DELETE',- 'path': path,- }-- if operation == 'write' and contents is not None:- # Convert bytes to string if needed- if isinstance(contents, bytes):- try:- # Try to decode as UTF-8- item['content'] = contents.decode('utf-8')- except UnicodeDecodeError:- # If not UTF-8, use base64 encoding- import base64-- item['content'] = base64.b64encode(contents).decode('ascii')- item['encoding'] = 'base64'- else:- item['content'] = contents-- batch_payload.append(item)-- # Send the batch as a single request- response = self.client.post(self.base_url, json=batch_payload)- response.raise_for_status()-- def flush(self) -> None:- """Immediately send any pending updates to the webhook.- This can be called to ensure all updates are sent before shutting down.- """- with self._batch_lock:- self._enqueue_batch_from_lock(background=False)diff --git a/openhands/storage/web_hook.py b/openhands/storage/web_hook.pydeleted file mode 100644index d41ef0b93b..0000000000--- a/openhands/storage/web_hook.py+++ /dev/null@@ -1,119 +0,0 @@-import httpx-import tenacity--from openhands.storage.files import FileStore-from openhands.utils.async_utils import EXECUTOR-from openhands.utils.http_session import httpx_verify_option---class WebHookFileStore(FileStore):- """File store which includes a web hook to be invoked after any changes occur.-- This class wraps another FileStore implementation and sends HTTP requests- to a specified URL whenever files are written or deleted.-- Attributes:- file_store: The underlying FileStore implementation- base_url: The base URL for webhook requests- client: The HTTP client used to make webhook requests- """-- file_store: FileStore- base_url: str- client: httpx.Client-- def __init__(- self, file_store: FileStore, base_url: str, client: httpx.Client | None = None- ):- """Initialize a WebHookFileStore.-- Args:- file_store: The underlying FileStore implementation- base_url: The base URL for webhook requests- client: Optional HTTP client to use for requests. If None, a new client will be created.- """- self.file_store = file_store- self.base_url = base_url- if client is None:- client = httpx.Client(verify=httpx_verify_option())- self.client = client-- def write(self, path: str, contents: str | bytes) -> None:- """Write contents to a file and trigger a webhook.-- Args:- path: The path to write to- contents: The contents to write- """- self.file_store.write(path, contents)- EXECUTOR.submit(self._on_write, path, contents)-- def read(self, path: str) -> str:- """Read contents from a file.-- Args:- path: The path to read from-- Returns:- The contents of the file- """- return self.file_store.read(path)-- def list(self, path: str) -> list[str]:- """List files in a directory.-- Args:- path: The directory path to list-- Returns:- A list of file paths- """- return self.file_store.list(path)-- def delete(self, path: str) -> None:- """Delete a file and trigger a webhook.-- Args:- path: The path to delete- """- self.file_store.delete(path)- EXECUTOR.submit(self._on_delete, path)-- @tenacity.retry(- wait=tenacity.wait_fixed(1),- stop=tenacity.stop_after_attempt(3),- )- def _on_write(self, path: str, contents: str | bytes) -> None:- """Send a POST request to the webhook URL when a file is written.-- This method is retried up to 3 times with a 1-second delay between attempts.-- Args:- path: The path that was written to- contents: The contents that were written-- Raises:- httpx.HTTPStatusError: If the webhook request fails- """- base_url = self.base_url + path- response = self.client.post(base_url, content=contents)- response.raise_for_status()-- @tenacity.retry(- wait=tenacity.wait_fixed(1),- stop=tenacity.stop_after_attempt(3),- )- def _on_delete(self, path: str) -> None:- """Send a DELETE request to the webhook URL when a file is deleted.-- This method is retried up to 3 times with a 1-second delay between attempts.-- Args:- path: The path that was deleted-- Raises:- httpx.HTTPStatusError: If the webhook request fails- """- base_url = self.base_url + path- response = self.client.delete(base_url)- response.raise_for_status()diff --git a/tests/unit/storage/settings/test_file_settings_store.py b/tests/unit/storage/settings/test_file_settings_store.pyindex 12b093a09f..8d00ef46e7 100644--- a/tests/unit/storage/settings/test_file_settings_store.py+++ b/tests/unit/storage/settings/test_file_settings_store.py@@ -118,7 +118,4 @@ async def test_get_instance():mock_get_store.assert_called_once_with(file_store_type='local',file_store_path='/test/path',- file_store_web_hook_url=None,- file_store_web_hook_headers=None,- file_store_web_hook_batch=False,)diff --git a/tests/unit/storage/test_batched_web_hook.py b/tests/unit/storage/test_batched_web_hook.pydeleted file mode 100644index 77322793b3..0000000000--- a/tests/unit/storage/test_batched_web_hook.py+++ /dev/null@@ -1,235 +0,0 @@-import time-from unittest.mock import MagicMock--import httpx-import pytest--from openhands.storage.batched_web_hook import BatchedWebHookFileStore-from openhands.storage.files import FileStore---class MockFileStore(FileStore):- def __init__(self):- self.files = {}-- def write(self, path: str, contents: str | bytes) -> None:- self.files[path] = contents-- def read(self, path: str) -> str:- return self.files.get(path, '')-- def list(self, path: str) -> list[str]:- return [k for k in self.files.keys() if k.startswith(path)]-- def delete(self, path: str) -> None:- if path in self.files:- del self.files[path]---class TestBatchedWebHookFileStore:- @pytest.fixture- def mock_client(self):- client = MagicMock(spec=httpx.Client)- client.post.return_value.raise_for_status = MagicMock()- client.delete.return_value.raise_for_status = MagicMock()- return client-- @pytest.fixture- def file_store(self):- return MockFileStore()-- @pytest.fixture- def batched_store(self, file_store, mock_client):- # Use a short timeout for testing- return BatchedWebHookFileStore(- file_store=file_store,- base_url='http://example.com',- client=mock_client,- batch_timeout_seconds=0.1, # Short timeout for testing- batch_size_limit_bytes=1000,- )-- def test_write_operation_batched(self, batched_store, mock_client):- # Write a file- batched_store.write('/test.txt', 'Hello, world!')-- # The client should not have been called yet- mock_client.post.assert_not_called()-- # Wait for the batch timeout- time.sleep(0.2)-- # Now the client should have been called with a batch payload- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1- assert batch_payload[0]['method'] == 'POST'- assert batch_payload[0]['path'] == '/test.txt'- assert batch_payload[0]['content'] == 'Hello, world!'-- def test_delete_operation_batched(self, batched_store, mock_client):- # Write and then delete a file- batched_store.write('/test.txt', 'Hello, world!')- batched_store.delete('/test.txt')-- # The client should not have been called yet- mock_client.post.assert_not_called()-- # Wait for the batch timeout- time.sleep(0.2)-- # Now the client should have been called with a batch payload- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1- assert batch_payload[0]['method'] == 'DELETE'- assert batch_payload[0]['path'] == '/test.txt'- assert 'content' not in batch_payload[0]-- def test_batch_size_limit_triggers_send(self, batched_store, mock_client):- # Write a large file that exceeds the batch size limit- large_content = 'x' * 1001 # Exceeds the 1000 byte limit- batched_store.write('/large.txt', large_content)-- # The batch might be sent asynchronously, so we need to wait a bit- time.sleep(0.2)-- # The client should have been called due to size limit- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1- assert batch_payload[0]['method'] == 'POST'- assert batch_payload[0]['path'] == '/large.txt'- assert batch_payload[0]['content'] == large_content-- def test_multiple_updates_same_file(self, batched_store, mock_client):- # Write to the same file multiple times- batched_store.write('/test.txt', 'Version 1')- batched_store.write('/test.txt', 'Version 2')- batched_store.write('/test.txt', 'Version 3')-- # Wait for the batch timeout- time.sleep(0.2)-- # Only the latest version should be sent- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1- assert batch_payload[0]['method'] == 'POST'- assert batch_payload[0]['path'] == '/test.txt'- assert batch_payload[0]['content'] == 'Version 3'-- def test_flush_sends_immediately(self, batched_store, mock_client):- # Write a file- batched_store.write('/test.txt', 'Hello, world!')-- # The client should not have been called yet- mock_client.post.assert_not_called()-- # Flush the batch- batched_store.flush()-- # Now the client should have been called without waiting for timeout- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1- assert batch_payload[0]['method'] == 'POST'- assert batch_payload[0]['path'] == '/test.txt'- assert batch_payload[0]['content'] == 'Hello, world!'-- def test_multiple_operations_in_single_batch(self, batched_store, mock_client):- # Perform multiple operations- batched_store.write('/file1.txt', 'Content 1')- batched_store.write('/file2.txt', 'Content 2')- batched_store.delete('/file3.txt')-- # Wait for the batch timeout- time.sleep(0.2)-- # Check that only one POST request was made with all operations- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 3-- # Check each operation in the batch- operations = {item['path']: item for item in batch_payload}-- assert '/file1.txt' in operations- assert operations['/file1.txt']['method'] == 'POST'- assert operations['/file1.txt']['content'] == 'Content 1'-- assert '/file2.txt' in operations- assert operations['/file2.txt']['method'] == 'POST'- assert operations['/file2.txt']['content'] == 'Content 2'-- assert '/file3.txt' in operations- assert operations['/file3.txt']['method'] == 'DELETE'- assert 'content' not in operations['/file3.txt']-- def test_binary_content_handling(self, batched_store, mock_client):- # Write binary content- binary_content = b'\x00\x01\x02\x03\xff\xfe\xfd\xfc'- batched_store.write('/binary.bin', binary_content)-- # Wait for the batch timeout- time.sleep(0.2)-- # Check that the client was called- mock_client.post.assert_called_once()- args, kwargs = mock_client.post.call_args- assert args[0] == 'http://example.com'- assert 'json' in kwargs-- # Check the batch payload- batch_payload = kwargs['json']- assert isinstance(batch_payload, list)- assert len(batch_payload) == 1-- # Binary content should be base64 encoded- assert batch_payload[0]['method'] == 'POST'- assert batch_payload[0]['path'] == '/binary.bin'- assert 'content' in batch_payload[0]- assert 'encoding' in batch_payload[0]- assert batch_payload[0]['encoding'] == 'base64'-- # Verify the content can be decoded back to the original binary- import base64-- decoded = base64.b64decode(batch_payload[0]['content'].encode('ascii'))- assert decoded == binary_content