π OpenHands: AI-Driven Development
Refactor event store cleanup (#8505)
tofarr7e88d4195f83
diff --git a/openhands/events/event_filter.py b/openhands/events/event_filter.pynew file mode 100644index 0000000000..cfd41b568c--- /dev/null+++ b/openhands/events/event_filter.py@@ -0,0 +1,98 @@+import json+from dataclasses import dataclass++from openhands.events.event import Event+from openhands.events.serialization.event import event_to_dict+++@dataclass+class EventFilter:+ """A filter for Event objects in the event stream.++ EventFilter provides a flexible way to filter events based on various criteria+ such as event type, source, date range, and content. It can be used to include+ or exclude events from search results based on the specified criteria.++ Attributes:+ exclude_hidden: Whether to exclude events marked as hidden. Defaults to False.+ query: Text string to search for in event content. Case-insensitive. Defaults to None.+ include_types: Tuple of Event types to include. Only events of these types will pass the filter.+ Defaults to None (include all types).+ exclude_types: Tuple of Event types to exclude. Events of these types will be filtered out.+ Defaults to None (exclude no types).+ source: Filter by event source (e.g., 'agent', 'user', 'environment'). Defaults to None.+ start_date: ISO format date string. Only events after this date will pass the filter.+ Defaults to None.+ end_date: ISO format date string. Only events before this date will pass the filter.+ Defaults to None.+ """++ exclude_hidden: bool = False+ query: str | None = None+ include_types: tuple[type[Event], ...] | None = None+ exclude_types: tuple[type[Event], ...] | None = None+ source: str | None = None+ start_date: str | None = None+ end_date: str | None = None++ def include(self, event: Event) -> bool:+ """Determine if an event should be included based on the filter criteria.++ This method checks if the given event matches all the filter criteria.+ If any criterion fails, the event is excluded.++ Args:+ event: The Event object to check against the filter criteria.++ Returns:+ bool: True if the event passes all filter criteria and should be included,+ False otherwise.+ """+ if self.include_types and not isinstance(event, self.include_types):+ return False++ if self.exclude_types is not None and isinstance(event, self.exclude_types):+ return False++ if self.source:+ if event.source is None or event.source.value != self.source:+ return False++ if (+ self.start_date+ and event.timestamp is not None+ and event.timestamp < self.start_date+ ):+ return False++ if (+ self.end_date+ and event.timestamp is not None+ and event.timestamp > self.end_date+ ):+ return False++ if self.exclude_hidden and getattr(event, 'hidden', False):+ return False++ # Text search in event content if query provided+ if self.query:+ event_dict = event_to_dict(event)+ event_str = json.dumps(event_dict).lower()+ if self.query.lower() not in event_str:+ return False++ return True++ def exclude(self, event: Event) -> bool:+ """Determine if an event should be excluded based on the filter criteria.++ This is the inverse of the include method.++ Args:+ event: The Event object to check against the filter criteria.++ Returns:+ bool: True if the event should be excluded, False if it should be included.+ """+ return not self.include(event)diff --git a/openhands/events/event_store.py b/openhands/events/event_store.pyindex 4b89ef8d67..fe094d66a8 100644--- a/openhands/events/event_store.py+++ b/openhands/events/event_store.py@@ -4,7 +4,9 @@ from typing import Iterablefrom openhands.core.logger import openhands_logger as loggerfrom openhands.events.event import Event, EventSource-from openhands.events.serialization.event import event_from_dict, event_to_dict+from openhands.events.event_filter import EventFilter+from openhands.events.event_store_abc import EventStoreABC+from openhands.events.serialization.event import event_from_dictfrom openhands.storage.files import FileStorefrom openhands.storage.locations import (get_conversation_dir,@@ -39,7 +41,7 @@ _DUMMY_PAGE = _CachePage(None, 1, -1)@dataclass-class EventStore:+class EventStore(EventStoreABC):"""A stored list of events backing a conversation"""@@ -60,15 +62,6 @@ class EventStore:except FileNotFoundError:logger.debug(f'No events found for session {self.sid} at {events_dir}')- if self.user_id:- # During transition to new location, try old location if user_id is set- # TODO: remove this code after 5/1/2025- try:- events_dir = get_conversation_events_dir(self.sid)- events += self.file_store.list(events_dir)- except FileNotFoundError:- logger.debug(f'No events found for session {self.sid} at {events_dir}')-if not events:self.cur_id = 0return@@ -79,13 +72,12 @@ class EventStore:if id >= self.cur_id:self.cur_id = id + 1- def get_events(+ def search_events(self,start_id: int = 0,end_id: int | None = None,reverse: bool = False,- filter_out_type: tuple[type[Event], ...] | None = None,- filter_hidden: bool = False,+ filter: EventFilter | None = None,) -> Iterable[Event]:"""Retrieve events from the event stream, optionally filtering out events of a given type@@ -95,20 +87,12 @@ class EventStore:start_id: The ID of the first event to retrieve. Defaults to 0.end_id: The ID of the last event to retrieve. Defaults to the last event in the stream.reverse: Whether to retrieve events in reverse order. Defaults to False.- filter_out_type: A tuple of event types to filter out. Typically used to filter out backend events from the agent.- filter_hidden: If True, filters out events with the 'hidden' attribute set to True.+ filter: EventFilter to useYields:Events from the stream that match the criteria."""- def should_filter(event: Event) -> bool:- if filter_hidden and hasattr(event, 'hidden') and event.hidden:- return True- if filter_out_type is not None and isinstance(event, filter_out_type):- return True- return False-if end_id is None:end_id = self.cur_idelse:@@ -134,24 +118,15 @@ class EventStore:event = self.get_event(index)except FileNotFoundError:event = None- if event and not should_filter(event):- yield event+ if event:+ if not filter or filter.include(event):+ yield eventdef get_event(self, id: int) -> Event:filename = self._get_filename_for_id(id, self.user_id)- try:- content = self.file_store.read(filename)- data = json.loads(content)- return event_from_dict(data)- except FileNotFoundError:- logger.debug(f'File {filename} not found')- # TODO remove this block after 5/1/2025- if self.user_id:- filename = self._get_filename_for_id(id, None)- content = self.file_store.read(filename)- data = json.loads(content)- return event_from_dict(data)- raise+ content = self.file_store.read(filename)+ data = json.loads(content)+ return event_from_dict(data)def get_latest_event(self) -> Event:return self.get_event(self.cur_id - 1)@@ -164,98 +139,6 @@ class EventStore:if event.source == source:yield event- def _should_filter_event(- self,- event: Event,- query: str | None = None,- event_types: tuple[type[Event], ...] | None = None,- source: str | None = None,- start_date: str | None = None,- end_date: str | None = None,- ) -> bool:- """Check if an event should be filtered out based on the given criteria.-- Args:- event: The event to check- query: Text to search for in event content- event_type: Filter by event type classes (e.g., (FileReadAction, ) ).- source: Filter by event source- start_date: Filter events after this date (ISO format)- end_date: Filter events before this date (ISO format)-- Returns:- bool: True if the event should be filtered out, False if it matches all criteria- """- if event_types and not isinstance(event, event_types):- return True-- if source:- if event.source is None or event.source.value != source:- return True-- if start_date and event.timestamp is not None and event.timestamp < start_date:- return True-- if end_date and event.timestamp is not None and event.timestamp > end_date:- return True-- # Text search in event content if query provided- if query:- event_dict = event_to_dict(event)- event_str = json.dumps(event_dict).lower()- if query.lower() not in event_str:- return True-- return False-- def get_matching_events(- self,- query: str | None = None,- event_types: tuple[type[Event], ...] | None = None,- source: str | None = None,- start_date: str | None = None,- end_date: str | None = None,- start_id: int = 0,- limit: int = 100,- reverse: bool = False,- ) -> list[Event]:- """Get matching events from the event stream based on filters.-- Args:- query: Text to search for in event content- event_types: Filter by event type classes (e.g., (FileReadAction, ) ).- source: Filter by event source- start_date: Filter events after this date (ISO format)- end_date: Filter events before this date (ISO format)- start_id: Starting ID in the event stream. Defaults to 0- limit: Maximum number of events to return. Must be between 1 and 100. Defaults to 100- reverse: Whether to retrieve events in reverse order. Defaults to False.-- Returns:- list: List of matching events (as dicts)-- Raises:- ValueError: If limit is less than 1 or greater than 100- """- if limit < 1 or limit > 100:- raise ValueError('Limit must be between 1 and 100')-- matching_events: list = []-- for event in self.get_events(start_id=start_id, reverse=reverse):- if self._should_filter_event(- event, query, event_types, source, start_date, end_date- ):- continue-- matching_events.append(event)-- # Stop if we have enough events- if len(matching_events) >= limit:- break-- return matching_events-def _get_filename_for_id(self, id: int, user_id: str | None) -> str:return get_conversation_event_filename(self.sid, id, user_id)diff --git a/openhands/events/event_store_abc.py b/openhands/events/event_store_abc.pynew file mode 100644index 0000000000..52e523e482--- /dev/null+++ b/openhands/events/event_store_abc.py@@ -0,0 +1,113 @@+from abc import abstractmethod+from itertools import islice+from typing import Iterable++from deprecated import deprecated # type: ignore++from openhands.events.event import Event, EventSource+from openhands.events.event_filter import EventFilter+++class EventStoreABC:+ """+ A stored list of events backing a conversation+ """++ sid: str+ user_id: str | None++ @abstractmethod+ def search_events(+ self,+ start_id: int = 0,+ end_id: int | None = None,+ reverse: bool = False,+ filter: EventFilter | None = None,+ ) -> Iterable[Event]:+ """+ Retrieve events from the event stream, optionally excluding events using a filter++ Args:+ start_id: The ID of the first event to retrieve. Defaults to 0.+ end_id: The ID of the last event to retrieve. Defaults to the last event in the stream.+ reverse: Whether to retrieve events in reverse order. Defaults to False.+ filter: An optional event filter++ Yields:+ Events from the stream that match the criteria.+ """++ @deprecated('Use search_events instead')+ def get_events(+ self,+ start_id: int = 0,+ end_id: int | None = None,+ reverse: bool = False,+ filter_out_type: tuple[type[Event], ...] | None = None,+ filter_hidden: bool = False,+ ) -> Iterable[Event]:+ yield from self.search_events(+ start_id,+ end_id,+ reverse,+ EventFilter(exclude_types=filter_out_type, exclude_hidden=filter_hidden),+ )++ @abstractmethod+ def get_event(self, id: int) -> Event:+ """Retrieve a single event from the event stream. Raise a FileNotFoundError if there was no such event"""++ @abstractmethod+ def get_latest_event(self) -> Event:+ """Get the latest event from the event stream"""++ @abstractmethod+ def get_latest_event_id(self) -> int:+ """Get the id of the latest event from the event stream"""++ @deprecated('use search_events instead')+ def filtered_events_by_source(self, source: EventSource) -> Iterable[Event]:+ yield from self.search_events(filter=EventFilter(source=source))++ @deprecated('use search_events instead')+ def get_matching_events(+ self,+ query: str | None = None,+ event_types: tuple[type[Event], ...] | None = None,+ source: str | None = None,+ start_date: str | None = None,+ end_date: str | None = None,+ start_id: int = 0,+ limit: int = 100,+ reverse: bool = False,+ ) -> list[Event]:+ """Get matching events from the event stream based on filters.++ Args:+ query: Text to search for in event content+ event_types: Filter by event type classes (e.g., (FileReadAction, ) ).+ source: Filter by event source+ start_date: Filter events after this date (ISO format)+ end_date: Filter events before this date (ISO format)+ start_id: Starting ID in the event stream. Defaults to 0+ limit: Maximum number of events to return. Must be between 1 and 100. Defaults to 100+ reverse: Whether to retrieve events in reverse order. Defaults to False.++ Returns:+ list: List of matching events (as dicts)+ """+ if limit < 1 or limit > 100:+ raise ValueError('Limit must be between 1 and 100')++ events = self.search_events(+ start_id=start_id,+ reverse=reverse,+ filter=EventFilter(+ query=query,+ include_types=event_types,+ source=source,+ start_date=start_date,+ end_date=end_date,+ ),+ )+ return list(islice(events, limit))diff --git a/tests/unit/test_event_stream.py b/tests/unit/test_event_stream.pyindex 9a3a5db72e..d5cf244678 100644--- a/tests/unit/test_event_stream.py+++ b/tests/unit/test_event_stream.py@@ -19,6 +19,7 @@ from openhands.events.action.files import ()from openhands.events.action.message import MessageActionfrom openhands.events.event import FileEditSource, FileReadSource+from openhands.events.event_filter import EventFilterfrom openhands.events.observation import NullObservationfrom openhands.events.observation.files import (FileEditObservation,@@ -177,12 +178,8 @@ def test_get_matching_events_source_filter(temp_dir: str):)# Verify that source comparison works correctly- assert event_stream._should_filter_event(- event, source='agent'- ) # Should filter out None source events- assert not event_stream._should_filter_event(- event, source=None- ) # Should not filter out when source filter is None+ assert EventFilter(source='agent').exclude(event)+ assert EventFilter(source=None).include(event)# Filter by AGENT source againevents = event_stream.get_matching_events(source='agent')