🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat(datagen): add stop_on_first_failure option to InstructionFilter (#1627)
Lei Zhang895db4f077fa
diff --git a/camel/datagen/self_instruct/filter/instruction_filter.py b/camel/datagen/self_instruct/filter/instruction_filter.pyindex 155cc1aa8..1df0a2bd5 100644--- a/camel/datagen/self_instruct/filter/instruction_filter.py+++ b/camel/datagen/self_instruct/filter/instruction_filter.py@@ -11,14 +11,22 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from typing import Any, Dict, List+from typing import Any, Dict, List, Tuple, Union++from camel.logger import get_loggerfrom .filter_function import FilterFunction, RewardModelFilterfrom .filter_registry import FILTER_REGISTRY+logger = get_logger(__name__)+class InstructionFilter:- def __init__(self, filters_config: Dict[str, Dict[str, Any]]):+ def __init__(+ self,+ filters_config: Dict[str, Dict[str, Any]],+ stop_on_first_failure: bool = False,+ ):r"""Initialize the InstructionFilter with a dictionary of filterconfigurations.@@ -37,12 +45,15 @@ class InstructionFilter:Each key in filters_config corresponds to a filter name(registered in FILTER_REGISTRY).Each value is a dict of parameters for that filter.+ stop_on_first_failure (bool): If True, stops checking filters after+ the first failure."""self.filters: List[FilterFunction] = []for filter_name, params in filters_config.items():if filter_name not in FILTER_REGISTRY:raise ValueError(f"Unknown filter function: {filter_name}")self.filters.append(FILTER_REGISTRY[filter_name](params))+ self.stop_on_first_failure: bool = stop_on_first_failuredef add_filter(self, filter_function: FilterFunction):r"""Add a custom filter function to the InstructionFilter.@@ -55,7 +66,7 @@ class InstructionFilter:def filter(self, prompt: str, instruction: str, return_details: bool = False- ):+ ) -> Union[bool, Tuple[bool, List[str]]]:r"""Check if the given instruction passes all filter functions.Args:@@ -75,6 +86,11 @@ class InstructionFilter:f.prompt = promptif not f.apply(instruction):failed_filters.append(type(f).__name__)+ logger.warning(+ f"{type(f).__name__} failed instruction: {instruction}"+ )+ if self.stop_on_first_failure:+ breakif return_details:return len(failed_filters) == 0, failed_filtersdiff --git a/camel/datagen/self_instruct/self_instruct.py b/camel/datagen/self_instruct/self_instruct.pyindex 80a964dc1..cd78a4f32 100644--- a/camel/datagen/self_instruct/self_instruct.py+++ b/camel/datagen/self_instruct/self_instruct.py@@ -45,6 +45,8 @@ class SelfInstructPipeline:filter_config (Optional[Dict[str, Dict[str, Any]]]): configurationfor the filter functions registered in FILE_REGISTRY.(default::obj:`None`)+ stop_on_first_failure (bool): If True, stops checking filters after+ the first failure."""def __init__(@@ -56,6 +58,7 @@ class SelfInstructPipeline:human_to_machine_ratio: tuple = (6, 2),instruction_filter: Optional[InstructionFilter] = None,filter_config: Optional[Dict[str, Dict[str, Any]]] = None,+ stop_on_first_failure: bool = False,):self.agent = agentself.num_machine_instructions = num_machine_instructions@@ -80,7 +83,9 @@ class SelfInstructPipeline:config_to_use = (filter_config if filter_config is not None else default_config)- self.instruction_filter = InstructionFilter(config_to_use)+ self.instruction_filter = InstructionFilter(+ config_to_use, stop_on_first_failure+ )def load_seed(self, path: str):r"""Load seed tasks from a file. Defaults to a predefined seed file ifdiff --git a/test/datagen/self_instruct/filter/instruction_filter_tests.py b/test/datagen/self_instruct/filter/instruction_filter_tests.pyindex bfb54c707..363479bad 100644--- a/test/datagen/self_instruct/filter/instruction_filter_tests.py+++ b/test/datagen/self_instruct/filter/instruction_filter_tests.py@@ -51,22 +51,55 @@ class TestInstructionFilter(unittest.TestCase):def test_all_pass_filters(self):filters_config = {"pass_filter": {}}instruction_filter = InstructionFilter(filters_config)- self.assertTrue(instruction_filter.filter("Any instruction"))+ self.assertTrue(+ instruction_filter.filter(prompt="", instruction="Any instruction")+ )result, failed_filters = instruction_filter.filter(- "Any instruction", return_details=True+ prompt="", instruction="Any instruction", return_details=True)self.assertTrue(result)self.assertEqual(len(failed_filters), 0)- @patch.dict(FILTER_REGISTRY, {"fail_filter": lambda _: DummyFailFilter()})+ @patch.dict(+ FILTER_REGISTRY,+ {+ "first_fail_filter": lambda _: DummyFailFilter(),+ "second_fail_filter": lambda _: DummyFailFilter(),+ },+ )def test_all_fail_filters(self):- filters_config = {"fail_filter": {}}+ filters_config = {"first_fail_filter": {}, "second_fail_filter": {}}instruction_filter = InstructionFilter(filters_config)- self.assertFalse(instruction_filter.filter("Any instruction"))+ self.assertFalse(+ instruction_filter.filter(prompt="", instruction="Any instruction")+ )result, failed_filters = instruction_filter.filter(- "Any instruction", return_details=True+ prompt="", instruction="Any instruction", return_details=True)self.assertFalse(result)+ self.assertEqual(len(failed_filters), 2)+ self.assertIn("DummyFailFilter", failed_filters)++ @patch.dict(+ FILTER_REGISTRY,+ {+ "first_fail_filter": lambda _: DummyFailFilter(),+ "second_fail_filter": lambda _: DummyFailFilter(),+ },+ )+ def test_all_fail_filters_early_stop(self):+ filters_config = {"first_fail_filter": {}, "second_fail_filter": {}}+ instruction_filter = InstructionFilter(+ filters_config, stop_on_first_failure=True+ )+ self.assertFalse(+ instruction_filter.filter(prompt="", instruction="Any instruction")+ )+ result, failed_filters = instruction_filter.filter(+ prompt="", instruction="Any instruction", return_details=True+ )+ self.assertFalse(result)+ self.assertEqual(len(failed_filters), 1)self.assertIn("DummyFailFilter", failed_filters)@patch.dict(@@ -85,27 +118,36 @@ class TestInstructionFilter(unittest.TestCase):def test_mixed_filters(self):instruction_filter = InstructionFilter(self.config_mixed)- self.assertTrue(instruction_filter.filter("This is valid"))+ self.assertTrue(+ instruction_filter.filter(prompt="", instruction="This is valid")+ )result, failed_filters = instruction_filter.filter(- "This is valid", return_details=True+ prompt="", instruction="This is valid", return_details=True)self.assertTrue(result)self.assertEqual(len(failed_filters), 0)self.assertFalse(instruction_filter.filter(- "This instruction is definitely too long"+ prompt="",+ instruction="This instruction is definitely too long",))result, failed_filters = instruction_filter.filter(- "This instruction is definitely too long", return_details=True+ prompt="",+ instruction="This instruction is definitely too long",+ return_details=True,)self.assertFalse(result)self.assertIn("MagicMock", failed_filters)- self.assertFalse(instruction_filter.filter("This is forbidden"))+ self.assertFalse(+ instruction_filter.filter(+ prompt="", instruction="This is forbidden"+ )+ )result, failed_filters = instruction_filter.filter(- "This is forbidden", return_details=True+ prompt="", instruction="This is forbidden", return_details=True)self.assertFalse(result)self.assertIn("MagicMock", failed_filters)@@ -119,12 +161,20 @@ class TestInstructionFilter(unittest.TestCase):def test_add_custom_filter(self):filters_config = {"pass_filter": {}}instruction_filter = InstructionFilter(filters_config)- self.assertTrue(instruction_filter.filter("Some instruction"))+ self.assertTrue(+ instruction_filter.filter(+ prompt="", instruction="Some instruction"+ )+ )instruction_filter.add_filter(DummyFailFilter())- self.assertFalse(instruction_filter.filter("Some instruction"))+ self.assertFalse(+ instruction_filter.filter(+ prompt="", instruction="Some instruction"+ )+ )result, failed_filters = instruction_filter.filter(- "Some instruction", return_details=True+ prompt="", instruction="Some instruction", return_details=True)self.assertFalse(result)self.assertIn("DummyFailFilter", failed_filters)