Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: python interpreter (#182)

Daniel Xuabff2ea951e0
diff --git a/camel/agents/embodied_agent.py b/camel/agents/embodied_agent.py
index 5a6dc8505..9343635f4 100644
--- a/camel/agents/embodied_agent.py
+++ b/camel/agents/embodied_agent.py
@@ -23,7 +23,7 @@ from camel.agents import (
)
from camel.messages import BaseMessage
from camel.typing import ModelType
-from camel.utils import print_text_animated
+from camel.utils import PythonInterpreter, print_text_animated
class EmbodiedAgent(ChatAgent):
@@ -120,12 +120,17 @@ class EmbodiedAgent(ChatAgent):
if codes is not None:
content = "\n> Executed Results:"
- global_vars = {action.name: action for action in self.action_space}
- for code in codes:
- executed_outputs = code.execute(global_vars)
- content += (
- f"- Python standard output:\n{executed_outputs[0]}\n"
- f"- Local variables:\n{executed_outputs[1]}\n")
+ action_space = {
+ action.name: action
+ for action in self.action_space
+ }
+ action_space.update({"print": print})
+ interpreter = PythonInterpreter(action_space=action_space)
+ for block_idx, code in enumerate(codes):
+ executed_outputs, _ = code.execute(interpreter)
+ content += (f"Executing code block {block_idx}:\n"
+ f" - execution output:\n{executed_outputs}\n"
+ f" - Local variables:\n{interpreter.state}\n")
content += "*" * 50 + "\n"
# TODO: Handle errors
diff --git a/camel/prompts/base.py b/camel/prompts/base.py
index a4bc72dce..cda2492fa 100644
--- a/camel/prompts/base.py
+++ b/camel/prompts/base.py
@@ -12,9 +12,20 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import inspect
-from typing import Any, Callable, Dict, Optional, Set, Tuple, TypeVar, Union
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ List,
+ Optional,
+ Set,
+ Tuple,
+ TypeVar,
+ Union,
+)
from camel.typing import RoleType
+from camel.utils import PythonInterpreter
T = TypeVar('T')
@@ -162,47 +173,31 @@ class CodePrompt(TextPrompt):
self._code_type = code_type
def execute(
- self,
- global_vars: Optional[Dict] = None) -> Tuple[str, Optional[Dict]]:
- r"""Executes the code string. If there is an error, the error is caught
- and the traceback is returned. Otherwise, the output string and local
- variables are returned.
+ self, interpreter: Optional[PythonInterpreter] = None,
+ user_variable: Optional[Dict[str, Any]] = None
+ ) -> Tuple[Any, PythonInterpreter]:
+ r"""Executes the code string by a given python interpreter.
Args:
- global_vars (Dict, optional): Global variables to be used during
- code execution. (default: :obj:`None`)
+ interpreter (PythonInterpreter, optional): interpreter to be used
+ during code execution. (default: :obj:`None`)
+ user_variable (Optional[Dict[str, Any]]): varibales that can be
+ used in the code, which applying fuzzy matching, such as images
+ or documents. (default: :obj:`None`)
Returns:
- Tuple[str, Optional[Dict]]: A tuple containing the output string
- and local variables.
- """
+ Tuple[Any, PythonInterpreter]: A tuple containing the execution
+ result and the used interpreter. The execution result
+ represents the value of the last statement (excluding "import")
+ in the code. This value could potentially be the desired result
+ of the LLM-generated code.
+ """
# NOTE: Only supports Python code for now.
- try:
- # Execute the code string
- import io
- import sys
- output_str = io.StringIO()
- sys.stdout = output_str
-
- global_vars = global_vars or globals()
- local_vars: Dict[str, Any] = {}
- exec(
- self,
- global_vars,
- local_vars,
- )
- sys.stdout = sys.__stdout__
- output_str.seek(0)
-
- # If there was no error, return the output and local variables
- return output_str.read(), local_vars
-
- except Exception:
- import traceback
- traceback_str = traceback.format_exc()
- sys.stdout = sys.__stdout__
- # If there was an error, return the traceback
- return traceback_str, None
+ if not interpreter:
+ interpreter = PythonInterpreter(action_space=globals())
+ execution_res = interpreter.execute(self, fuzz_state=user_variable,
+ keep_state=True)
+ return execution_res, interpreter
# flake8: noqa :E501
diff --git a/camel/utils/__init__.py b/camel/utils/__init__.py
new file mode 100644
index 000000000..f5b3f8530
--- /dev/null
+++ b/camel/utils/__init__.py
@@ -0,0 +1,36 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from .python_interpreter import PythonInterpreter
+from .functions import (
+ get_model_encoding,
+ count_tokens_openai_chat_models,
+ num_tokens_from_messages,
+ openai_api_key_required,
+ print_text_animated,
+ get_prompt_template_key_words,
+ get_first_int,
+ download_tasks,
+)
+
+__all__ = [
+ 'get_model_encoding',
+ 'count_tokens_openai_chat_models',
+ 'num_tokens_from_messages',
+ 'openai_api_key_required',
+ 'print_text_animated',
+ 'get_prompt_template_key_words',
+ 'get_first_int',
+ 'download_tasks',
+ 'PythonInterpreter',
+]
diff --git a/camel/utils.py b/camel/utils/functions.py
similarity index 100%
rename from camel/utils.py
rename to camel/utils/functions.py
diff --git a/camel/utils/python_interpreter.py b/camel/utils/python_interpreter.py
new file mode 100644
index 000000000..e9f0b8145
--- /dev/null
+++ b/camel/utils/python_interpreter.py
@@ -0,0 +1,403 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import ast
+import difflib
+import importlib
+from typing import Any, Dict, List, Mapping, Optional, Union
+
+
+class InterpreterError(ValueError):
+ r"""An error raised when the interpreter cannot evaluate a Python
+ expression, due to syntax error or unsupported operations.
+ """
+
+ pass
+
+
+class PythonInterpreter():
+ r"""A customized python interpreter to control the execution of
+ LLM-generated codes. The interpreter makes sure the code can only execute
+ functions given in action space and import white list. It also supports
+ fuzzy variable matching to reveive uncertain input variable name.
+
+ .. highlight:: none
+
+ This class is adapted from the hugging face implementation
+ `python_interpreter.py <https://github.com/huggingface/transformers/blob/8f
+ 093fb799246f7dd9104ff44728da0c53a9f67a/src/transformers/tools/python_interp
+ reter.py>`_. The original license applies::
+
+ Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied. See the License for the specific language governing
+ permissions and limitations under the License.
+
+ We have modified the original code to suit our requirements. We have
+ encapsulated the original functions within a class and saved the
+ interpreter state after execution. We have added support for "import"
+ statements, "for" statements, and several binary and unary operators. We
+ have added import white list to keep `import` statement safe. Additionally,
+ we have modified the variable matching logic and introduced the
+ :obj:`fuzz_state` for fuzzy matching.
+
+ Modifications copyright (C) 2023 CAMEL-AI.org
+
+ Args:
+ action_space (Dict[str, Any]): A dictionary that maps action names to
+ their corresponding functions or objects. The interpreter can only
+ execute functions that are either directly listed in this
+ dictionary or are member functions of objects listed in this
+ dictionary. The concept of :obj:`action_space` is derived from
+ EmbodiedAgent, representing the actions that an agent is capable of
+ performing.
+ import_white_list (Optional[List[str]], optional): A list that stores
+ the Python modules or functions that can be imported in the code.
+ All submodules and functions of the modules listed in this list are
+ importable. Any other import statements will be rejected. The
+ module and its submodule or function name are separated by a period
+ (:obj:`.`). (default: :obj:`None`)
+ """
+
+ def __init__(self, action_space: Dict[str, Any],
+ import_white_list: Optional[List[str]] = None) -> None:
+ self.action_space = action_space
+ self.state = self.action_space.copy()
+ self.fuzz_state = {}
+ self.import_white_list = import_white_list or []
+
+ def execute(self, code: str, state: Dict[str, Any] = None,
+ fuzz_state: Dict[str,
+ Any] = None, keep_state: bool = True) -> Any:
+ r""" Execute the input python codes in a security environment.
+
+ Args:
+ code (str): Generated python code to be executed.
+ state (Optional[Dict[str, Any]], optional): External variables that
+ may be used in the generated code. (default: :obj:`None`)
+ fuzz_state (Optional[Dict[str, Any]], optional): External varibles
+ that do not have certain varible names. The interpreter will
+ use fuzzy matching to access these varibales. For example, if
+ :obj:`fuzz_state` has a variable :obj:`image`, the generated
+ code can use :obj:`input_image` to access it. (default:
+ :obj:`None`)
+ keep_state (bool, optional): If :obj:`True`, :obj:`state` and
+ :obj:`fuzz_state` will be kept for later execution. Otherwise,
+ they will be cleared. (default: :obj:`True`)
+
+ Returns:
+ Any: The value of the last statement (excluding "import") in the
+ code. For this interpreter, the value of an expression is its
+ value, the value of an "assign" statement is the assigned
+ value, and the value of an "if" and "for" block statement is
+ the value of the last statement in the block.
+ """
+ if state is not None:
+ self.state.update(state)
+ if fuzz_state is not None:
+ self.fuzz_state.update(fuzz_state)
+
+ try:
+ expression = ast.parse(code)
+ except SyntaxError as e:
+ raise InterpreterError(f"Syntax error in code: {e}")
+
+ result = None
+ for idx, node in enumerate(expression.body):
+ try:
+ line_result = self._execute_ast(node)
+ except InterpreterError as e:
+ if not keep_state:
+ self.clear_state()
+ msg = (f"Evaluation of the code stopped at line {idx}. "
+ f"See:\n{e}")
+ # More information can be provided by `ast.unparse()`,
+ # which is new in python 3.9.
+ raise InterpreterError(msg)
+ if line_result is not None:
+ result = line_result
+
+ if not keep_state:
+ self.clear_state()
+
+ return result
+
+ def clear_state(self) -> None:
+ r"""Initialize :obj:`state` and :obj:`fuzz_state`"""
+ self.state = self.action_space.copy()
+ self.fuzz_state = {}
+
+ def _execute_ast(self, expression: ast.AST) -> Any:
+ if isinstance(expression, ast.Assign):
+ # Assignement -> evaluate the assignement which should
+ # update the state. We return the variable assigned as it may
+ # be used to determine the final result.
+ return self._execute_assign(expression)
+ elif isinstance(expression, ast.Attribute):
+ value = self._execute_ast(expression.value)
+ return getattr(value, expression.attr)
+ elif isinstance(expression, ast.BinOp):
+ # Binary Operator -> return the result value
+ return self._execute_binop(expression)
+ elif isinstance(expression, ast.Call):
+ # Function call -> return the value of the function call
+ return self._execute_call(expression)
+ elif isinstance(expression, ast.Constant):
+ # Constant -> just return the value
+ return expression.value
+ elif isinstance(expression, ast.Dict):
+ # Dict -> evaluate all keys and values
+ keys = [self._execute_ast(k) for k in expression.keys]
+ values = [self._execute_ast(v) for v in expression.values]
+ return dict(zip(keys, values))
+ elif isinstance(expression, ast.Expr):
+ # Expression -> evaluate the content
+ return self._execute_ast(expression.value)
+ elif isinstance(expression, ast.For):
+ return self._execute_for(expression)
+ elif isinstance(expression, ast.FormattedValue):
+ # Formatted value (part of f-string) -> evaluate the content
+ # and return
+ return self._execute_ast(expression.value)
+ elif isinstance(expression, ast.If):
+ # If -> execute the right branch
+ return self._execute_if(expression)
+ elif isinstance(expression, ast.Import):
+ # Import -> add imported names in self.state and return None.
+ for module in expression.names:
+ self._execute_import(module)
+ return None
+ elif isinstance(expression, ast.ImportFrom):
+ module = expression.module
+ for name in expression.names:
+ self._execute_import(module, name)
+ return None
+ elif hasattr(ast, "Index") and isinstance(expression, ast.Index):
+ return self._execute_ast(expression.value)
+ elif isinstance(expression, ast.JoinedStr):
+ return "".join(
+ [str(self._execute_ast(v)) for v in expression.values])
+ elif isinstance(expression, ast.List):
+ # List -> evaluate all elements
+ return [self._execute_ast(elt) for elt in expression.elts]
+ elif isinstance(expression, ast.Name):
+ # Name -> pick up the value in the state
+ return self._execute_name(expression)
+ elif isinstance(expression, ast.Subscript):
+ # Subscript -> return the value of the indexing
+ return self._execute_subscript(expression)
+ elif isinstance(expression, ast.Tuple):
+ return tuple([self._execute_ast(elt) for elt in expression.elts])
+ elif isinstance(expression, ast.UnaryOp):
+ # Binary Operator -> return the result value
+ return self._execute_unaryop(expression)
+ else:
+ # For now we refuse anything else. Let's add things as we need
+ # them.
+ raise InterpreterError(
+ f"{expression.__class__.__name__} is not supported.")
+
+ def _execute_assign(self, assign: ast.Assign) -> Any:
+ var_names = assign.targets
+ result = self._execute_ast(assign.value)
+
+ for var_name in var_names:
+ self._assign(var_name, result)
+ return result
+
+ def _assign(self, target: Union[ast.Name, ast.Tuple], value: Any):
+ if isinstance(target, ast.Name):
+ self.state[target.id] = value
+ elif isinstance(target, ast.Tuple):
+ if len(value) != len(target):
+ raise InterpreterError(f"Expected {len(target)} values but got"
+ f"{len(value)}.")
+ for v, r in zip(target.ctx, value):
+ self.state[v.id] = r
+ else:
+ raise InterpreterError(f"Unsupport variable type. Expected"
+ f"ast.Name or ast.Tuple, got"
+ f"{type(target)} instead.")
+
+ def _execute_call(self, call: ast.Call) -> Any:
+ callable_func = self._execute_ast(call.func)
+
+ # Todo deal with args
+ args = [self._execute_ast(arg) for arg in call.args]
+ kwargs = {
+ keyword.arg: self._execute_ast(keyword.value)
+ for keyword in call.keywords
+ }
+ return callable_func(*args, **kwargs)
+
+ def _execute_subscript(self, subscript):
+ index = self._execute_ast(subscript.slice)
+ value = self._execute_ast(subscript.value)
+ if isinstance(value, (list, tuple)):
+ return value[int(index)]
+ if index in value:
+ return value[index]
+ if isinstance(index, str) and isinstance(value, Mapping):
+ close_matches = difflib.get_close_matches(index,
+ list(value.keys()))
+ if len(close_matches) > 0:
+ return value[close_matches[0]]
+
+ raise InterpreterError(f"Could not index {value} with '{index}'.")
+
+ def _execute_name(self, name: ast.Name):
+ return self._get_value_from_state(name.id)
+
+ def _execute_condition(self, condition):
+ if len(condition.ops) > 1:
+ raise InterpreterError(
+ "Cannot evaluate conditions with multiple operators")
+
+ left = self._execute_ast(condition.left)
+ comparator = condition.ops[0]
+ right = self._execute_ast(condition.comparators[0])
+
+ if isinstance(comparator, ast.Eq):
+ return left == right
+ elif isinstance(comparator, ast.NotEq):
+ return left != right
+ elif isinstance(comparator, ast.Lt):
+ return left < right
+ elif isinstance(comparator, ast.LtE):
+ return left <= right
+ elif isinstance(comparator, ast.Gt):
+ return left > right
+ elif isinstance(comparator, ast.GtE):
+ return left >= right
+ elif isinstance(comparator, ast.Is):
+ return left is right
+ elif isinstance(comparator, ast.IsNot):
+ return left is not right
+ elif isinstance(comparator, ast.In):
+ return left in right
+ elif isinstance(comparator, ast.NotIn):
+ return left not in right
+ else:
+ raise InterpreterError(f"Operator not supported: {comparator}")
+
+ def _execute_if(self, if_statement: ast.If):
+ result = None
+ if self._execute_condition(if_statement.test):
+ for line in if_statement.body:
+ line_result = self._execute_ast(line)
+ if line_result is not None:
+ result = line_result
+ else:
+ for line in if_statement.orelse:
+ line_result = self._execute_ast(line)
+ if line_result is not None:
+ result = line_result
+ return result
+
+ def _execute_for(self, for_statement: ast.For):
+ result = None
+ for value in self._execute_ast(for_statement.iter):
+ self._assign(for_statement.target, value)
+ for line in for_statement.body:
+ line_result = self._execute_ast(line)
+ if line_result is not None:
+ result = line_result
+
+ return result
+
+ def _execute_import(self, module: Union[str, ast.alias],
+ import_name: Optional[ast.alias] = None) -> None:
+ module_name = module if isinstance(module, str) else module.name
+ found_name = False
+ tmp_name = ""
+ full_name = (module_name if import_name is None else module_name +
+ f".{import_name.name}")
+ for name in full_name.split("."):
+ tmp_name += name if tmp_name == "" else f".{name}"
+ if tmp_name in self.import_white_list:
+ found_name = True
+ break
+
+ if not found_name:
+ raise InterpreterError(f"It is not permitted to import modules "
+ f"than module white list (try to import "
+ f"{module_name}).")
+
+ if import_name is None:
+ alias = (module.asname or module_name)
+ self.state[alias] = importlib.import_module(module_name)
+ else:
+ imported_module = importlib.import_module(module_name)
+ alias = import_name.asname or import_name.name
+ self.state[alias] = getattr(imported_module, import_name.name)
+
+ def _execute_binop(self, binop: ast.BinOp):
+ left = self._execute_ast(binop.left)
+ operator = binop.op
+ right = self._execute_ast(binop.right)
+
+ if isinstance(operator, ast.Add):
+ return left + right
+ elif isinstance(operator, ast.Sub):
+ return left - right
+ elif isinstance(operator, ast.Mult):
+ return left * right
+ elif isinstance(operator, ast.Div):
+ return left / right
+ elif isinstance(operator, ast.FloorDiv):
+ return left // right
+ elif isinstance(operator, ast.Mod):
+ return left % right
+ elif isinstance(operator, ast.Pow):
+ return left**right
+ elif isinstance(operator, ast.LShift):
+ return left << right
+ elif isinstance(operator, ast.RShift):
+ return left >> right
+ elif isinstance(operator, ast.MatMult):
+ return left @ right
+ else:
+ raise InterpreterError(f"Operator not supported: {operator}")
+
+ def _execute_unaryop(self, unaryop: ast.UnaryOp):
+ operand = self._execute_ast(unaryop.operand)
+ operator = unaryop.op
+
+ if isinstance(operator, ast.UAdd):
+ return +operand
+ elif isinstance(operator, ast.USub):
+ return -operand
+ elif isinstance(operator, ast.Not):
+ return not operand
+ else:
+ raise InterpreterError(f"Operator not supported: {operator}")
+
+ def _get_value_from_state(self, key: str) -> Any:
+ if key in self.state:
+ return self.state[key]
+ else:
+ close_matches = (difflib.get_close_matches(
+ key, list(self.fuzz_state.keys()), n=1))
+ if close_matches:
+ return self.fuzz_state[close_matches[0]]
+ else:
+ raise InterpreterError(f"The variable `{key}` is not defined.")
diff --git a/test/prompts/test_prompt_base.py b/test/prompts/test_prompt_base.py
index 9c722c94a..ec77aeefe 100644
--- a/test/prompts/test_prompt_base.py
+++ b/test/prompts/test_prompt_base.py
@@ -11,6 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import pytest
+
from camel.prompts.base import (
CodePrompt,
TextPrompt,
@@ -18,6 +20,8 @@ from camel.prompts.base import (
return_prompt_wrapper,
wrap_prompt_functions,
)
+from camel.utils import PythonInterpreter
+from camel.utils.python_interpreter import InterpreterError
def test_return_prompt_wrapper():
@@ -135,13 +139,20 @@ def test_code_prompt_set_code_type():
assert code_prompt.code_type == "python"
-def test_code_prompt_execute():
+def test_code_prompt_execute(capsys):
code_prompt = CodePrompt("a = 1\nprint('Hello, World!')",
code_type="python")
- assert code_prompt.execute() == ("Hello, World!\n", {"a": 1})
+ interpreter = PythonInterpreter(action_space={"print": print})
+ result, interpreter = code_prompt.execute(interpreter=interpreter)
+ captured = capsys.readouterr()
+ assert result == 1
+ assert interpreter.state["a"] == 1
+ assert captured.out == "Hello, World!\n"
def test_code_prompt_execute_error():
code_prompt = CodePrompt("print('Hello, World!'", code_type="python")
- traceback_str, _ = code_prompt.execute()
- assert "SyntaxError" in traceback_str
+ interpreter = PythonInterpreter(action_space={"print": print})
+ with pytest.raises(InterpreterError) as e:
+ _, _ = code_prompt.execute(interpreter=interpreter)
+ assert e.value.args[0].startswith("Syntax error in code:")
diff --git a/test/utils/test_python_interpreter.py b/test/utils/test_python_interpreter.py
new file mode 100644
index 000000000..9e0d4f522
--- /dev/null
+++ b/test/utils/test_python_interpreter.py
@@ -0,0 +1,157 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import numpy as np
+import pytest
+import torch
+
+from camel.utils import PythonInterpreter
+from camel.utils.python_interpreter import InterpreterError
+
+
+def action_function():
+ return "access action function"
+
+
+def interpreter():
+ action_space = {"action1": action_function}
+ white_list = ["torch", "numpy.array"]
+ return PythonInterpreter(action_space=action_space,
+ import_white_list=white_list)
+
+
+def test_import_success0(interpreter):
+ code = """import torch
+a = torch.tensor([[1., -1.], [1., -1.]])"""
+ rnt = interpreter.execute(code)
+ assert torch.equal(interpreter.state["a"],
+ torch.tensor([[1., -1.], [1., -1.]]))
+ assert torch.equal(rnt, torch.tensor([[1., -1.], [1., -1.]]))
+
+
+def test_import_success1(interpreter):
+ code = """from torch import tensor
+a = tensor([[1., -1.], [1., -1.]])"""
+ rnt = interpreter.execute(code)
+ assert torch.equal(rnt, torch.tensor([[1., -1.], [1., -1.]]))
+
+
+def test_import_success2(interpreter):
+ code = """import torch as pytorch
+a = pytorch.tensor([[1., -1.], [1., -1.]])"""
+ rnt = interpreter.execute(code)
+ assert torch.equal(rnt, torch.tensor([[1., -1.], [1., -1.]]))
+
+
+def test_import_success3(interpreter):
+ code = """from numpy import array
+x = array([[1, 2, 3], [4, 5, 6]])"""
+ rnt = interpreter.execute(code)
+ assert np.equal(rnt, np.array([[1, 2, 3], [4, 5, 6]])).all()
+
+
+def test_import_fail0(interpreter):
+ code = """import os
+os.mkdir("/tmp/test")"""
+ with pytest.raises(InterpreterError) as e:
+ interpreter.execute(code)
+ exec_msg = e.value.args[0]
+ assert exec_msg == ("Evaluation of the code stopped at line 0. See:\n"
+ "It is not permitted to import modules than module"
+ " white list (try to import os).")
+
+
+def test_import_fail1(interpreter):
+ code = """import numpy as np
+x = np.array([[1, 2, 3], [4, 5, 6]], np.int32)"""
+ with pytest.raises(InterpreterError) as e:
+ interpreter.execute(code)
+ exec_msg = e.value.args[0]
+ assert exec_msg == ("Evaluation of the code stopped at line 0. See:\n"
+ "It is not permitted to import modules than module"
+ " white list (try to import numpy).")
+
+
+def test_action_space(interpreter):
+ code = "rnt = action1()"
+ rnt = interpreter.execute(code)
+ assert rnt == "access action function"
+
+
+def test_fuzz_space(interpreter):
+ from PIL import Image
+ fuzz_state = {"image": Image.new("RGB", (256, 256))}
+ code = "output_image = input_image.crop((20, 20, 100, 100))"
+ rnt = interpreter.execute(code, fuzz_state=fuzz_state)
+ assert rnt.width == 80
+ assert rnt.height == 80
+
+
+def test_keep_state0(interpreter):
+ code1 = "a = 42"
+ code2 = "b = a"
+ code3 = "c = b"
+
+ rnt = interpreter.execute(code1, keep_state=True)
+ assert rnt == 42
+ rnt = interpreter.execute(code2, keep_state=False)
+ assert rnt == 42
+ with pytest.raises(InterpreterError) as e:
+ interpreter.execute(code3, keep_state=False)
+ exec_msg = e.value.args[0]
+ assert exec_msg == ("Evaluation of the code stopped at line 0. See:\n"
+ "The variable `b` is not defined.")
+
+
+def test_keep_state1(interpreter):
+ code1 = "from torch import tensor"
+ code2 = "a = tensor([[1., -1.], [1., -1.]])"
+ rnt = interpreter.execute(code1, keep_state=True)
+ rnt = interpreter.execute(code2, keep_state=False)
+ assert torch.equal(rnt, torch.tensor([[1., -1.], [1., -1.]]))
+ with pytest.raises(InterpreterError) as e:
+ interpreter.execute(code2, keep_state=False)
+ exec_msg = e.value.args[0]
+ assert exec_msg == ("Evaluation of the code stopped at line 0. See:\n"
+ "The variable `tensor` is not defined.")
+
+
+def test_if(interpreter):
+ code = """a = 0
+b = 1
+if a < b:
+ t = a
+ a = b
+ b = t
+else:
+ b = a"""
+ interpreter.execute(code)
+ assert interpreter.state["a"] == 1
+ assert interpreter.state["b"] == 0
+
+
+def test_for(interpreter):
+ code = """l = [2, 3, 5, 7, 11]
+sum = 0
+for i in l:
+ sum = sum + i"""
+ rnt = interpreter.execute(code)
+ assert rnt == 28
+
+
+def test_subscript(interpreter):
+ code = """l = [2, 3, 5, 7, 11]
+rnt = l[3]"""
+ rnt = interpreter.execute(code)
+ assert rnt == 7