Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat(datagen): add stop_on_first_failure option to InstructionFilter (#1627)

Lei Zhang895db4f077fa
diff --git a/camel/datagen/self_instruct/filter/instruction_filter.py b/camel/datagen/self_instruct/filter/instruction_filter.py
index 155cc1aa8..1df0a2bd5 100644
--- a/camel/datagen/self_instruct/filter/instruction_filter.py
+++ b/camel/datagen/self_instruct/filter/instruction_filter.py
@@ -11,14 +11,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple, Union
+
+from camel.logger import get_logger
from .filter_function import FilterFunction, RewardModelFilter
from .filter_registry import FILTER_REGISTRY
+logger = get_logger(__name__)
+
class InstructionFilter:
- def __init__(self, filters_config: Dict[str, Dict[str, Any]]):
+ def __init__(
+ self,
+ filters_config: Dict[str, Dict[str, Any]],
+ stop_on_first_failure: bool = False,
+ ):
r"""Initialize the InstructionFilter with a dictionary of filter
configurations.
@@ -37,12 +45,15 @@ class InstructionFilter:
Each key in filters_config corresponds to a filter name
(registered in FILTER_REGISTRY).
Each value is a dict of parameters for that filter.
+ stop_on_first_failure (bool): If True, stops checking filters after
+ the first failure.
"""
self.filters: List[FilterFunction] = []
for filter_name, params in filters_config.items():
if filter_name not in FILTER_REGISTRY:
raise ValueError(f"Unknown filter function: {filter_name}")
self.filters.append(FILTER_REGISTRY[filter_name](params))
+ self.stop_on_first_failure: bool = stop_on_first_failure
def add_filter(self, filter_function: FilterFunction):
r"""Add a custom filter function to the InstructionFilter.
@@ -55,7 +66,7 @@ class InstructionFilter:
def filter(
self, prompt: str, instruction: str, return_details: bool = False
- ):
+ ) -> Union[bool, Tuple[bool, List[str]]]:
r"""Check if the given instruction passes all filter functions.
Args:
@@ -75,6 +86,11 @@ class InstructionFilter:
f.prompt = prompt
if not f.apply(instruction):
failed_filters.append(type(f).__name__)
+ logger.warning(
+ f"{type(f).__name__} failed instruction: {instruction}"
+ )
+ if self.stop_on_first_failure:
+ break
if return_details:
return len(failed_filters) == 0, failed_filters
diff --git a/camel/datagen/self_instruct/self_instruct.py b/camel/datagen/self_instruct/self_instruct.py
index 80a964dc1..cd78a4f32 100644
--- a/camel/datagen/self_instruct/self_instruct.py
+++ b/camel/datagen/self_instruct/self_instruct.py
@@ -45,6 +45,8 @@ class SelfInstructPipeline:
filter_config (Optional[Dict[str, Dict[str, Any]]]): configuration
for the filter functions registered in FILE_REGISTRY.
(default::obj:`None`)
+ stop_on_first_failure (bool): If True, stops checking filters after
+ the first failure.
"""
def __init__(
@@ -56,6 +58,7 @@ class SelfInstructPipeline:
human_to_machine_ratio: tuple = (6, 2),
instruction_filter: Optional[InstructionFilter] = None,
filter_config: Optional[Dict[str, Dict[str, Any]]] = None,
+ stop_on_first_failure: bool = False,
):
self.agent = agent
self.num_machine_instructions = num_machine_instructions
@@ -80,7 +83,9 @@ class SelfInstructPipeline:
config_to_use = (
filter_config if filter_config is not None else default_config
)
- self.instruction_filter = InstructionFilter(config_to_use)
+ self.instruction_filter = InstructionFilter(
+ config_to_use, stop_on_first_failure
+ )
def load_seed(self, path: str):
r"""Load seed tasks from a file. Defaults to a predefined seed file if
diff --git a/test/datagen/self_instruct/filter/instruction_filter_tests.py b/test/datagen/self_instruct/filter/instruction_filter_tests.py
index bfb54c707..363479bad 100644
--- a/test/datagen/self_instruct/filter/instruction_filter_tests.py
+++ b/test/datagen/self_instruct/filter/instruction_filter_tests.py
@@ -51,22 +51,55 @@ class TestInstructionFilter(unittest.TestCase):
def test_all_pass_filters(self):
filters_config = {"pass_filter": {}}
instruction_filter = InstructionFilter(filters_config)
- self.assertTrue(instruction_filter.filter("Any instruction"))
+ self.assertTrue(
+ instruction_filter.filter(prompt="", instruction="Any instruction")
+ )
result, failed_filters = instruction_filter.filter(
- "Any instruction", return_details=True
+ prompt="", instruction="Any instruction", return_details=True
)
self.assertTrue(result)
self.assertEqual(len(failed_filters), 0)
- @patch.dict(FILTER_REGISTRY, {"fail_filter": lambda _: DummyFailFilter()})
+ @patch.dict(
+ FILTER_REGISTRY,
+ {
+ "first_fail_filter": lambda _: DummyFailFilter(),
+ "second_fail_filter": lambda _: DummyFailFilter(),
+ },
+ )
def test_all_fail_filters(self):
- filters_config = {"fail_filter": {}}
+ filters_config = {"first_fail_filter": {}, "second_fail_filter": {}}
instruction_filter = InstructionFilter(filters_config)
- self.assertFalse(instruction_filter.filter("Any instruction"))
+ self.assertFalse(
+ instruction_filter.filter(prompt="", instruction="Any instruction")
+ )
result, failed_filters = instruction_filter.filter(
- "Any instruction", return_details=True
+ prompt="", instruction="Any instruction", return_details=True
)
self.assertFalse(result)
+ self.assertEqual(len(failed_filters), 2)
+ self.assertIn("DummyFailFilter", failed_filters)
+
+ @patch.dict(
+ FILTER_REGISTRY,
+ {
+ "first_fail_filter": lambda _: DummyFailFilter(),
+ "second_fail_filter": lambda _: DummyFailFilter(),
+ },
+ )
+ def test_all_fail_filters_early_stop(self):
+ filters_config = {"first_fail_filter": {}, "second_fail_filter": {}}
+ instruction_filter = InstructionFilter(
+ filters_config, stop_on_first_failure=True
+ )
+ self.assertFalse(
+ instruction_filter.filter(prompt="", instruction="Any instruction")
+ )
+ result, failed_filters = instruction_filter.filter(
+ prompt="", instruction="Any instruction", return_details=True
+ )
+ self.assertFalse(result)
+ self.assertEqual(len(failed_filters), 1)
self.assertIn("DummyFailFilter", failed_filters)
@patch.dict(
@@ -85,27 +118,36 @@ class TestInstructionFilter(unittest.TestCase):
def test_mixed_filters(self):
instruction_filter = InstructionFilter(self.config_mixed)
- self.assertTrue(instruction_filter.filter("This is valid"))
+ self.assertTrue(
+ instruction_filter.filter(prompt="", instruction="This is valid")
+ )
result, failed_filters = instruction_filter.filter(
- "This is valid", return_details=True
+ prompt="", instruction="This is valid", return_details=True
)
self.assertTrue(result)
self.assertEqual(len(failed_filters), 0)
self.assertFalse(
instruction_filter.filter(
- "This instruction is definitely too long"
+ prompt="",
+ instruction="This instruction is definitely too long",
)
)
result, failed_filters = instruction_filter.filter(
- "This instruction is definitely too long", return_details=True
+ prompt="",
+ instruction="This instruction is definitely too long",
+ return_details=True,
)
self.assertFalse(result)
self.assertIn("MagicMock", failed_filters)
- self.assertFalse(instruction_filter.filter("This is forbidden"))
+ self.assertFalse(
+ instruction_filter.filter(
+ prompt="", instruction="This is forbidden"
+ )
+ )
result, failed_filters = instruction_filter.filter(
- "This is forbidden", return_details=True
+ prompt="", instruction="This is forbidden", return_details=True
)
self.assertFalse(result)
self.assertIn("MagicMock", failed_filters)
@@ -119,12 +161,20 @@ class TestInstructionFilter(unittest.TestCase):
def test_add_custom_filter(self):
filters_config = {"pass_filter": {}}
instruction_filter = InstructionFilter(filters_config)
- self.assertTrue(instruction_filter.filter("Some instruction"))
+ self.assertTrue(
+ instruction_filter.filter(
+ prompt="", instruction="Some instruction"
+ )
+ )
instruction_filter.add_filter(DummyFailFilter())
- self.assertFalse(instruction_filter.filter("Some instruction"))
+ self.assertFalse(
+ instruction_filter.filter(
+ prompt="", instruction="Some instruction"
+ )
+ )
result, failed_filters = instruction_filter.filter(
- "Some instruction", return_details=True
+ prompt="", instruction="Some instruction", return_details=True
)
self.assertFalse(result)
self.assertIn("DummyFailFilter", failed_filters)