π OpenHands: AI-Driven Development
test: Add MC/DC tests for loop pattern detector (stuck_detector) (#11600)
Pedro Henriquea3e85e2c2daa
diff --git a/tests/unit/controller/test_is_stuck.py b/tests/unit/controller/test_is_stuck.pyindex 09e2c4c02c..838ba28651 100644--- a/tests/unit/controller/test_is_stuck.py+++ b/tests/unit/controller/test_is_stuck.py@@ -7,8 +7,13 @@ from pytest import TempPathFactoryfrom openhands.controller.agent_controller import AgentControllerfrom openhands.controller.state.state import Statefrom openhands.controller.stuck import StuckDetector-from openhands.events.action import CmdRunAction, FileReadAction, MessageAction+from openhands.events.action import (+ CmdRunAction,+ FileReadAction,+ MessageAction,+)from openhands.events.action.commands import IPythonRunCellAction+from openhands.events.event import Eventfrom openhands.events.observation import (CmdOutputObservation,FileReadObservation,@@ -20,6 +25,28 @@ from openhands.events.observation.error import ErrorObservationfrom openhands.events.stream import EventSource, EventStreamfrom openhands.storage import get_file_store+# Reusable action/observation mocks for stuck-pattern tests+cmd_ls_action = CmdRunAction(command='ls')+cmd_ls_observation = CmdOutputObservation(command='ls', content='file1.txt')++read_file1_action = FileReadAction(path='file1.txt')+read_file1_observation = FileReadObservation(content='File content', path='file1.txt')++pwd_action = CmdRunAction(command='pwd')+read_file2_action = FileReadAction(path='file2.txt')++cmd_ls_different_observation = CmdOutputObservation(+ command='ls_DIFFERENT', content='file1.txt'+)+read_file2_observation = FileReadObservation(content='File content', path='file2.txt')+++class MockOtherEvent(Event):+ """Event type used to verify non-action/observation events are ignored."""+++other_event = MockOtherEvent()+def collect_events(stream):return [event for event in stream.get_events()]@@ -757,10 +784,11 @@ class TestStuckDetector:assert stuck_detector.stuck_analysis.loop_start_idx == 0def test_is_not_stuck_context_window_error_in_non_headless(self, stuck_detector):- """Test that in non-headless mode, we don't detect a loop if the condensation events- are before the last user message.+ """Test non-headless mode with condensation events before last user message.- In non-headless mode, we only look at events after the last user message.+ In non-headless mode, we don't detect a loop if the condensation events+ are before the last user message. We only look at events after the last+ user message."""state = stuck_detector.state@@ -793,6 +821,196 @@ class TestStuckDetector:assert stuck_detector.is_stuck(headless_mode=False) is Falsemock_warning.assert_not_called()+ @pytest.fixture+ def stuck_detector_mcdc(self):+ return StuckDetector(state=None)++ def test_fail_guard_five_actions(self, stuck_detector_mcdc: StuckDetector):+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1, obs_1, action_2, obs_2+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3, obs_3, action_4, obs_4+ read_file1_action,+ read_file1_observation, # action_5, obs_5+ ] # Total 5 actions, 5 observations+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_fail_guard_five_obs(self, stuck_detector_mcdc: StuckDetector):+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1, obs_1, action_2, obs_2+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3, obs_3, action_4, obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action, # action_5, obs_5, action_6 (missing obs_6)+ ] # Total 6 actions, 5 observations+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_fail_actions_break_A6_A4(self, stuck_detector_mcdc: StuckDetector):+ # action_6(cmd_ls_action) != action_4(pwd_action)+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1,obs_1, action_2,obs_2+ read_file1_action,+ read_file1_observation,+ pwd_action,+ cmd_ls_observation, # action_3,obs_3, action_4(pwd_action),obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5,obs_5, action_6(cmd_ls_action),obs_6+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_fail_actions_break_A5_A3(self, stuck_detector_mcdc: StuckDetector):+ # action_5(read_file1_action) != action_3(read_file2_action)+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1,obs_1, action_2,obs_2+ read_file2_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3(read_file2_action),obs_3, action_4,obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5(read_file1_action),obs_5, action_6,obs_6+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_fail_obs_break_O6_O4(self, stuck_detector_mcdc: StuckDetector):+ # obs_6(cmd_ls_observation) != obs_4(cmd_ls_different_observation)+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1,obs_1, action_2,obs_2+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_different_observation, # action_3,obs_3, action_4,obs_4(cmd_ls_different_observation)+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5,obs_5, action_6,obs_6(cmd_ls_observation)+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_fail_obs_break_O5_O3(self, stuck_detector_mcdc: StuckDetector):+ # obs_5(read_file1_observation) != obs_3(read_file2_observation)+ history: list[Event] = [+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1,obs_1, action_2,obs_2+ read_file1_action,+ read_file2_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3,obs_3(read_file2_observation), action_4,obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5(read_file1_observation),obs_5, action_6,obs_6+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0)+ is False+ )++ def test_loop_ignores_other_events(self, stuck_detector_mcdc: StuckDetector):+ history: list[Event] = [+ other_event,+ read_file1_action,+ read_file1_observation,+ other_event, # action_1, obs_1+ cmd_ls_action,+ cmd_ls_observation,+ other_event,+ read_file1_action,+ read_file1_observation, # action_2, obs_2, action_3, obs_3+ cmd_ls_action,+ cmd_ls_observation,+ other_event,+ read_file1_action,+ read_file1_observation, # action_4, obs_4, action_5, obs_5+ cmd_ls_action,+ cmd_ls_observation,+ other_event, # action_6, obs_6+ ]++ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0) is True+ )++ def test_loop_ignores_extra_actions(self, stuck_detector_mcdc: StuckDetector):+ history: list[Event] = [+ cmd_ls_action, # Extra action (action_0), will be ignored+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1, obs_1, action_2, obs_2+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3, obs_3, action_4, obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5, obs_5, action_6, obs_6+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0) is True+ )++ def test_loop_ignores_extra_observations(self, stuck_detector_mcdc: StuckDetector):+ history: list[Event] = [+ read_file1_observation, # Extra observation (obs_0), will be ignored+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_1, obs_1, action_2, obs_2+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_3, obs_3, action_4, obs_4+ read_file1_action,+ read_file1_observation,+ cmd_ls_action,+ cmd_ls_observation, # action_5, obs_5, action_6, obs_6+ ]+ assert (+ stuck_detector_mcdc._is_stuck_action_observation_pattern(history, 0) is True+ )+class TestAgentController:@pytest.fixture