🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: implement Tic Tac Toe as `MultiStepEnv` (#1914)
hallerite5069b893896f
diff --git a/camel/environments/__init__.py b/camel/environments/__init__.pyindex c580dd577..cfd925736 100644--- a/camel/environments/__init__.py+++ b/camel/environments/__init__.py@@ -14,6 +14,7 @@from .models import Action, Environment, Observation, StepResultfrom .multi_step import MultiStepEnvfrom .single_step import SingleStepEnv+from .tic_tac_toe import Opponent, TicTacToeEnv__all__ = ["Environment",@@ -22,4 +23,6 @@ __all__ = ["Action","Observation","StepResult",+ "TicTacToeEnv",+ "Opponent",]diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.pyindex df0013954..e73b2af6d 100644--- a/camel/environments/multi_step.py+++ b/camel/environments/multi_step.py@@ -15,11 +15,10 @@from abc import ABC, abstractmethodfrom typing import Any, Dict, List, Optional, Tuple+from camel.environments.models import Action, Observation, StepResultfrom camel.extractors.base import BaseExtractorfrom camel.logger import get_logger-from .models import Action, Observation, StepResult-logger = get_logger(__name__)@@ -73,9 +72,8 @@ class MultiStepEnv(ABC):logger.error(f'Failed to setup environment: {e}')raise- @abstractmethodasync def _setup(self) -> None:- pass+ returnasync def close(self) -> None:r"""Clean up and close all resources used by the environment.@@ -100,9 +98,8 @@ class MultiStepEnv(ABC):logger.error(f'Failed to teardown environment: {e}')raise- @abstractmethodasync def _close(self) -> None:- pass+ returnasync def reset(self) -> Observation:r"""Reset the environment to an initial state.@@ -115,6 +112,9 @@ class MultiStepEnv(ABC):"""if not self._is_setup:+ logger.warning(+ "reset() called on un-setup environment. Setting up..."+ )await self.setup()# Reset state@@ -132,7 +132,9 @@ class MultiStepEnv(ABC):return observation- async def step(self, action: Action) -> StepResult:+ async def step(+ self, action: Action+ ) -> Tuple[Observation, float, bool, Dict[str, Any]]:r"""Take a step in the environment using the given action.This method updates the environment state based on the LLM's response,@@ -157,7 +159,7 @@ class MultiStepEnv(ABC):rewards_dict={},done=True,info={"reason": "max_steps_reached"},- )+ ).as_tuple()if not self._is_setup:raise RuntimeError("Environment not set up. Call setup() first.")@@ -196,13 +198,13 @@ class MultiStepEnv(ABC):rewards_dict=rewards_dict,done=done,info={- "extraction_result": self.extractor.extract(+ "extraction_result": await self.extractor.extract(action.llm_response),"step": self._current_step,"state": self._state, # Updated state},- )+ ).as_tuple()@abstractmethoddef _get_initial_state(self) -> Dict[str, Any]:diff --git a/camel/environments/tic_tac_toe.py b/camel/environments/tic_tac_toe.pynew file mode 100644index 000000000..be6f17896--- /dev/null+++ b/camel/environments/tic_tac_toe.py@@ -0,0 +1,518 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import math+import random+import re+from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple++from camel.environments.models import Action, Observation+from camel.environments.multi_step import MultiStepEnv+from camel.extractors import BaseExtractor, BaseExtractorStrategy+++class MoveExtractor(BaseExtractorStrategy):+ r"""A strategy for extracting Tic Tac Toe actions from text."""++ async def extract(self, text: str) -> Optional[str]:+ r"""Extract a valid Tic Tac Toe move from text.++ Looks for a pattern '<Action> n' where n is a digit between 1 and 9.++ Args:+ text (str): The text to extract the action from.++ Returns:+ Optional[str]: The extracted move as a string, or None if no valid+ move is found.+ """+ match = re.search(r"<Action>\s*(\d+)", text)+ if match:+ move = match.group(1)+ # Validate that the move is in range 1-9+ if move.isdigit() and 1 <= int(move) <= 9:+ return move+ return None+++class Opponent:+ r"""AI opponent for the Tic Tac Toe game.++ This class implements different playing strategies for the AI opponent,+ including an optimal strategy using the minimax algorithm with alpha-beta+ pruning, and a random strategy.+ """++ def __init__(+ self, play_style: Literal["optimal", "random"] = "optimal"+ ) -> None:+ r"""Initialize the opponent with a specific play style.++ Args:+ play_style (Literal["optimal", "random"]): The strategy to use,+ either "optimal" or "random". (default: :obj:`"optimal"`)+ """+ self.play_style = play_style++ def select_move(self, board: List[str]) -> Optional[int]:+ r"""Select a move based on the opponent's play style.++ Args:+ board (List[str]): The current game board as a list of strings.++ Returns:+ Optional[int]: The index of the selected move, or None if no move+ is available.+ """+ if self.play_style == "optimal":+ return self.get_optimal_move(board)+ elif self.play_style == "random":+ moves = TicTacToeEnv.available_moves(board)+ if not moves:+ return None # Consistent with optimal strategy+ return random.choice(moves)++ def get_optimal_move(self, board: List[str]) -> Optional[int]:+ r"""Get the optimal move using the minimax algorithm.++ Args:+ board (List[str]): The current game board as a list of strings.++ Returns:+ Optional[int]: The index of the optimal move, or None if no move+ is available.+ """+ _, move = self.minimax(board, is_maximizing=True)+ return move++ def minimax(+ self,+ board: List[str],+ is_maximizing: bool,+ depth: int = 0,+ alpha: float = -math.inf,+ beta: float = math.inf,+ ) -> Tuple[float, Optional[int]]:+ r"""Minimax algorithm with alpha-beta pruning for optimal move+ selection.++ Recursively evaluates all possible moves to find the best one.+ Uses alpha-beta pruning to reduce the search space.++ Args:+ board (List[str]): The current game board as a list of strings.+ is_maximizing (bool): True if maximizing player (O), False if+ minimizing (X).+ depth (int): Current depth in the search tree. (default: :obj:`0`)+ alpha (float): Alpha value for pruning. (default: :obj:`-math.inf`)+ beta (float): Beta value for pruning. (default: :obj:`math.inf`)++ Returns:+ Tuple[float, Optional[int]]: A tuple containing:+ - float: The score of the best move (1 for O win, -1 for X+ win, 0 for draw)+ - Optional[int]: The index of the best move, or None if+ terminal state+ """+ winner = TicTacToeEnv.check_winner(board)+ if winner == "O":+ return (1, None)+ elif winner == "X":+ return (-1, None)+ elif winner == "draw":+ return (0, None)++ moves = TicTacToeEnv.available_moves(board)+ # Add depth limit to prevent stack overflow (9 is max depth for+ # tic-tac-toe)+ if depth >= 9:+ # Evaluate current position+ return (0, None)++ if is_maximizing:+ best_score = -math.inf+ best_move = None+ for move in moves:+ board[move] = "O"+ score, _ = self.minimax(+ board,+ is_maximizing=False,+ depth=depth + 1,+ alpha=alpha,+ beta=beta,+ )+ board[move] = " "+ if score > best_score:+ best_score = score+ best_move = move+ alpha = max(alpha, best_score)+ if beta <= alpha:+ break # Beta cutoff+ return best_score, best_move+ else:+ best_score = math.inf+ best_move = None+ for move in moves:+ board[move] = "X"+ score, _ = self.minimax(+ board,+ is_maximizing=True,+ depth=depth + 1,+ alpha=alpha,+ beta=beta,+ )+ board[move] = " "+ if score < best_score:+ best_score = score+ best_move = move+ beta = min(beta, best_score)+ if beta <= alpha:+ break # Alpha cutoff+ return best_score, best_move+++class TicTacToeEnv(MultiStepEnv):+ r"""A Tic Tac Toe environment for reinforcement learning with LLMs.++ This environment implements a standard Tic Tac Toe game where the LLM agent+ plays as 'X' against an AI opponent that plays as 'O'. The opponent can use+ either an optimal strategy (minimax with alpha-beta pruning) or a random+ strategy.+ """++ WIN_COMBINATIONS: ClassVar = [+ (0, 1, 2), # Top row+ (3, 4, 5), # Middle row+ (6, 7, 8), # Bottom row+ (0, 3, 6), # Left column+ (1, 4, 7), # Middle column+ (2, 5, 8), # Right column+ (0, 4, 8), # Diagonal from top-left+ (2, 4, 6), # Diagonal from top-right+ ]++ def __init__(+ self,+ extractor: Optional[BaseExtractor] = None,+ max_steps: Optional[int] = None,+ play_style: Literal["optimal", "random"] = "optimal",+ **kwargs,+ ) -> None:+ r"""Initialize the Tic Tac Toe environment.++ Args:+ extractor (Optional[BaseExtractor]): Extractor to process LLM+ responses. If None, a default extractor with+ MoveExtractor will be used. (default: :obj:`None`)+ max_steps (Optional[int]): Maximum steps per episode.+ (default: :obj:`None`)+ play_style (Literal["optimal", "random"]): The strategy for the+ opponent to use, either "optimal" or "random". (default:+ :obj:`"optimal"`)+ **kwargs: Additional environment parameters.+ """+ if extractor is None:+ extractor = BaseExtractor(pipeline=[[MoveExtractor()]])+ super().__init__(extractor, max_steps, **kwargs)+ self.opponent = Opponent(play_style=play_style)++ def _get_initial_state(self) -> Dict[str, Any]:+ r"""Get the initial state of the environment.++ Returns:+ Dict[str, Any]: A dictionary containing the initial state with an+ empty board, game status flags, and move history.+ """+ # State includes the board (9 cells), game_over flag, and winner info.+ return {+ "board": [" " for _ in range(9)],+ "game_over": False,+ "winner": None,+ "last_move_illegal": False,+ "last_move": None,+ }++ async def _update_state(self, action: Action) -> None:+ r"""Update the environment state based on the agent's action.++ This method processes the agent's move, updates the board, checks for+ a winner, and if the game is not over, makes a move for the opponent.++ Args:+ action (Action): The action containing the LLM's response with the+ chosen move.++ Returns:+ None+ """+ board = self._state["board"]++ # Attempt to parse the agent's chosen move+ extraction_result = await self.extractor.extract(action.llm_response)+ if not extraction_result:+ # Handle extraction failure gracefully+ self._state["last_move_illegal"] = True+ self._state["last_move"] = None+ self._state["extraction_error"] = "Could not extract a valid move"+ return++ try:+ move = int(extraction_result)+ self._state["last_move"] = move+ self._state["extraction_error"] = None+ except ValueError:+ # Handle invalid move format gracefully+ self._state["last_move_illegal"] = True+ self._state["last_move"] = extraction_result+ self._state["extraction_error"] = (+ f"'{extraction_result}' is not a valid number"+ )+ return++ # Convert 1-indexed move to 0-indexed board position.+ index = move - 1+ if index < 0 or index > 8 or board[index] != " ":+ self._state["last_move_illegal"] = True+ self._state["extraction_error"] = (+ f"Position {move} is not a valid or available move"+ )+ return++ # Reset the flag+ self._state["last_move_illegal"] = False++ # Agent (X) makes the move.+ board[index] = "X"++ # Check if agent wins (or draw) right after its move.+ winner = self.check_winner(board)+ if winner is not None:+ self._state["game_over"] = True+ self._state["winner"] = winner+ return++ # Opponent (O) plays using the opponent class.+ opponent_move = self.opponent.select_move(board)+ if opponent_move is not None:+ board[opponent_move] = "O"++ # Check if the game ended after opponent's move.+ winner = self.check_winner(board)+ if winner is not None:+ self._state["game_over"] = True+ self._state["winner"] = winner++ def _get_next_observation(self) -> Observation:+ r"""Get the next observation based on the current state.++ This method generates a text observation describing the current state+ of the game board and prompting the agent to make a move.++ Returns:+ Observation: An Observation object containing the game state+ description.+ """+ board = self._state["board"]+ if self._state["last_move_illegal"]:+ obs = (+ "You are playing Tic Tac Toe with standard rules.\n"+ "You are the player with X.\n"+ "Your last move was illegal.\n"+ f"You chose the move {self._state['last_move']}."+ "Choose another number between 1 and 9 to place an X.\n"+ "The field must still be available.\n"+ "This is the current state of the board:\n"+ f"{self.render_board(board)}\n"+ "Each number that you can see is still an empty field "+ "that you can place your 'X' in. Please end your response "+ "with <Action> [a number from 1 to 9]"+ )+ else:+ obs = (+ "You are playing Tic Tac Toe with standard rules.\n"+ "You are the player with X.\n"+ "Choose a number between 1 and 9 to place an X.\n"+ "This is the current state of the board:\n"+ f"{self.render_board(board)}\n"+ "Each number that you can see is still an empty field "+ "that you can place your 'X' in. Please end your response "+ "with <Action> [a number from 1 to 9]"+ )++ return Observation(question=obs, context={}, metadata={})++ def _get_terminal_observation(self) -> Observation:+ r"""Get the final observation when the game is over.++ This method generates a text observation describing the final state+ of the game board and the game result (win, loss, or draw).++ Returns:+ Observation: An Observation object containing the final game state+ description.+ """+ board = self._state["board"]+ result_message = ""+ if self._state["winner"] == "X":+ result_message = "Congratulations, you won!"+ elif self._state["winner"] == "O":+ result_message = "Sorry, you lost!"+ else:+ result_message = "It's a draw!"++ obs = f"{self.render_board(board)}\nGame Over. {result_message}"++ return Observation(question=obs, context={}, metadata={})++ async def compute_reward(self) -> Tuple[float, Dict[str, float]]:+ r"""Compute the reward for the current state.++ Returns:+ Tuple[float, Dict[str, float]]: A tuple containing the total+ reward and a dictionary of reward components:+ - 1.0 for a win+ - 0.0 for a loss or illegal move+ - 0.5 for a draw+ - For ongoing games, returns an evaluation of the position+ """+ # Simple reward: 1 for win, 0 for loss, 0.5 for draw or ongoing.+ if self._state["game_over"]:+ if self._state["winner"] == "X":+ return 1.0, {"win": 1.0}+ elif self._state["winner"] == "O":+ return 0.0, {"loss": 0.0}+ else:+ return 0.5, {"draw": 0.5}++ elif self._state["last_move_illegal"]:+ return 0.0, {"illegal_move": 0.0}++ else:+ board = self._state["board"]+ value = TicTacToeEnv.evaluate_position_for_x(board, is_x_turn=True)+ return value, {"x_non_loss_value": value}++ @staticmethod+ def evaluate_position_for_x(+ board: List[str], is_x_turn: bool, depth: int = 0, max_depth: int = 10+ ) -> float:+ r"""Evaluate the current board position from X's perspective.++ Uses minimax to determine the value of the position.++ Args:+ board (List[str]): The current game board as a list of strings.+ is_x_turn (bool): True if it's X's turn to move, False otherwise.++ Returns:+ float: A float value representing the position evaluation:+ - 1.0 if X has a winning position+ - 0.0 if O has a winning position+ - 0.5 for a draw+ - For ongoing positions, returns the expected outcome with+ perfect play+ """+ winner = TicTacToeEnv.check_winner(board)+ if winner == "X":+ return 1.0 # X wins+ elif winner == "O":+ return 0.0 # X loses+ elif winner == "draw":+ return 0.5 # draw++ # Add depth limit to prevent potential stack overflow+ if depth >= max_depth:+ return 0.5 # Return draw evaluation at max depth++ moves = TicTacToeEnv.available_moves(board)+ values = []+ # Create a copy of the board to avoid side effects+ for move in moves:+ board_copy = board.copy()+ board_copy[move] = "X" if is_x_turn else "O"+ value = TicTacToeEnv.evaluate_position_for_x(+ board_copy, not is_x_turn, depth + 1, max_depth+ )+ values.append(value)++ return max(values) if is_x_turn else min(values)++ def _is_done(self) -> bool:+ r"""Check if the episode is done.++ Returns:+ True if the game is over, False otherwise.+ """+ return self._state["game_over"]++ @staticmethod+ def available_moves(board: List[str]) -> List[int]:+ r"""Get all available moves on the board.++ Args:+ board (List[str]): The current game board as a list of strings.++ Returns:+ List[int]: A list of indices representing empty cells on the board.+ """+ # Return list of indices that are free.+ return [i for i, cell in enumerate(board) if cell == " "]++ @staticmethod+ def check_winner(board: List[str]) -> Optional[Literal["X", "O", "draw"]]:+ r"""Check if there is a winner or a draw on the board.++ Args:+ board (List[str]): The current game board as a list of strings.++ Returns:+ Optional[Literal["X", "O", "draw"]]: "X" if X has won, "O" if O+ has won, "draw" if the game is a draw, or None if the game is+ still ongoing.+ """+ # Check all win combinations.+ for a, b, c in TicTacToeEnv.WIN_COMBINATIONS:+ if board[a] != " " and board[a] == board[b] == board[c]:+ return board[a]+ # Check for draw.+ if all(cell != " " for cell in board):+ return "draw"+ return None++ def render_board(self, board: List[str]) -> str:+ r"""Render the board as a string for display.++ Args:+ board (List[str]): The current game board as a list of strings.++ Returns:+ str: A formatted string representation of the board.+ """++ # Create a nice formatted board.+ def cell_value(i: int) -> str:+ r"""Get the display value for a cell.++ Args:+ i (int): The index of the cell.++ Returns:+ str: The cell content ("X" or "O") or the cell number if empty.+ """+ return board[i] if board[i] != " " else str(i + 1)++ rows = []+ for i in range(0, 9, 3):+ row = " | ".join(cell_value(j) for j in range(i, i + 3))+ rows.append(row)+ return "\n---------\n".join(rows)diff --git a/docs/cookbooks/loong/multi_step_rl.ipynb b/docs/cookbooks/loong/multi_step_rl.ipynbnew file mode 100644index 000000000..58cd47c5d--- /dev/null+++ b/docs/cookbooks/loong/multi_step_rl.ipynb@@ -0,0 +1,206 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/home/student/true-gnosis/Code/camel/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Initial Observation:\n",+ "\n",+ "You are playing Tic Tac Toe with standard rules.\n",+ "You are the player with X.\n",+ "Choose a number between 1 and 9 to place an X.\n",+ "This is the current state of the board:\n",+ "1 | 2 | 3\n",+ "---------\n",+ "4 | 5 | 6\n",+ "---------\n",+ "7 | 8 | 9\n",+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n"+ ]+ }+ ],+ "source": [+ "import asyncio\n",+ "from camel.environments.models import Action\n",+ "from camel.environments.tic_tac_toe import TicTacToeEnv, Opponent # Adjust if needed\n",+ "\n",+ "# Initialize and set up the environment\n",+ "env = TicTacToeEnv(opponent=Opponent(play_style=\"random\"))\n",+ "await env.setup()\n",+ "\n",+ "# Reset environment and get initial observation\n",+ "observation = await env.reset()\n",+ "print(\"Initial Observation:\\n\")\n",+ "print(observation.question)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "import os\n",+ "from getpass import getpass\n",+ "\n",+ "openai_api_key = getpass('Enter your API key: ')\n",+ "os.environ[\"OPENAI_API_KEY\"] = openai_api_key"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "from camel.models import ModelFactory\n",+ "from camel.types import ModelPlatformType, ModelType\n",+ "from camel.configs import ChatGPTConfig\n",+ "from camel.agents import ChatAgent\n",+ "\n",+ "model = ModelFactory.create(\n",+ " model_platform=ModelPlatformType.OPENAI,\n",+ " model_type=ModelType.GPT_4O_MINI,\n",+ " model_config_dict=ChatGPTConfig().as_dict(),\n",+ ")\n",+ "\n",+ "agent = ChatAgent(model=model, system_message=\"PLEASE RESPOND WITH <Action>\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "Agent Move: index=0 llm_response=\"I'll place my X in position 5. \\n\\n<Action> 5\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 47, 647354, tzinfo=datetime.timezone.utc)\n",+ "Observation:\n",+ "You are playing Tic Tac Toe with standard rules.\n",+ "You are the player with X.\n",+ "Choose a number between 1 and 9 to place an X.\n",+ "This is the current state of the board:\n",+ "O | 2 | 3\n",+ "---------\n",+ "4 | X | 6\n",+ "---------\n",+ "7 | 8 | 9\n",+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",+ "Reward: 0.5\n",+ "Done: False\n",+ "Info: {'extraction_result': '5', 'step': 1, 'state': {'board': ['O', ' ', ' ', ' ', 'X', ' ', ' ', ' ', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 5}, 'rewards_dict': {'x_non_loss_value': 0.5}}\n",+ "\n",+ "Agent Move: index=0 llm_response=\"I'll place my X in position 2.\\n\\n<Action> 2\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 48, 369680, tzinfo=datetime.timezone.utc)\n",+ "Observation:\n",+ "You are playing Tic Tac Toe with standard rules.\n",+ "You are the player with X.\n",+ "Choose a number between 1 and 9 to place an X.\n",+ "This is the current state of the board:\n",+ "O | X | 3\n",+ "---------\n",+ "4 | X | 6\n",+ "---------\n",+ "7 | O | 9\n",+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",+ "Reward: 0.5\n",+ "Done: False\n",+ "Info: {'extraction_result': '2', 'step': 2, 'state': {'board': ['O', 'X', ' ', ' ', 'X', ' ', ' ', 'O', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 2}, 'rewards_dict': {'x_non_loss_value': 0.5}}\n",+ "\n",+ "Agent Move: index=0 llm_response=\"I'll place my X in position 3.\\n\\n<Action> 3\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 49, 25743, tzinfo=datetime.timezone.utc)\n",+ "Observation:\n",+ "You are playing Tic Tac Toe with standard rules.\n",+ "You are the player with X.\n",+ "Choose a number between 1 and 9 to place an X.\n",+ "This is the current state of the board:\n",+ "O | X | X\n",+ "---------\n",+ "4 | X | 6\n",+ "---------\n",+ "O | O | 9\n",+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",+ "Reward: 0.0\n",+ "Done: False\n",+ "Info: {'extraction_result': '3', 'step': 3, 'state': {'board': ['O', 'X', 'X', ' ', 'X', ' ', 'O', 'O', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 3}, 'rewards_dict': {'x_non_loss_value': 0.0}}\n",+ "\n",+ "Agent Move: index=0 llm_response=\"I'll place my X in position 6.\\n\\n<Action> 6\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 49, 588802, tzinfo=datetime.timezone.utc)\n",+ "Observation:\n",+ "O | X | X\n",+ "---------\n",+ "O | X | X\n",+ "---------\n",+ "O | O | 9\n",+ "Game Over. Sorry, you lost!\n",+ "Reward: 0.0\n",+ "Done: True\n",+ "Info: {'extraction_result': '6', 'step': 4, 'state': {'board': ['O', 'X', 'X', 'O', 'X', 'X', 'O', 'O', ' '], 'game_over': True, 'winner': 'O', 'last_move_illegal': False, 'last_move': 6}, 'rewards_dict': {'loss': 0.0}}\n"+ ]+ }+ ],+ "source": [+ "while not env.is_done():\n",+ "\n",+ " llm_response = agent.step(observation.question).msgs[0].content\n",+ " #agent.reset()\n",+ "\n",+ " action = Action(llm_response=llm_response)\n",+ " result = await env.step(action)\n",+ "\n",+ " observation, reward, done, info = result\n",+ "\n",+ " print(\"\\nAgent Move:\", action)\n",+ " print(\"Observation:\")\n",+ " print(observation.question)\n",+ " print(\"Reward:\", reward)\n",+ " print(\"Done:\", done)\n",+ " print(\"Info:\", info)\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Close the environment\n",+ "await env.close()"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": ".venv",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.11"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/examples/environments/tic_tac_toe.py b/examples/environments/tic_tac_toe.pynew file mode 100644index 000000000..db7105226--- /dev/null+++ b/examples/environments/tic_tac_toe.py@@ -0,0 +1,57 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncio+import random++from camel.environments import TicTacToeEnv+from camel.environments.models import Action+++async def run_game():+ env = TicTacToeEnv()+ # Initialize and set up the environment+ await env.setup()++ # Reset environment and get initial observation+ observation = await env.reset()+ print("Initial Observation:\n")+ print(observation.question)++ while not env.is_done():+ available_moves = [+ move + 1+ for move in TicTacToeEnv.available_moves(env._state["board"])+ ]++ action = Action(+ llm_response=f"<Action> {random.choice(available_moves)}"+ )+ result = await env.step(action)++ observation, reward, done, info = result++ print("\nAgent Move:", action)+ print("Observation:")+ print(observation.question)+ print("Reward:", reward)+ print("Done:", done)+ print("Info:", info)++ print("\n___________\n\nBoard at the end:\n")+ print(env.render_board(env._state["board"]), "\n")+ await env._close()+++if __name__ == "__main__":+ asyncio.run(run_game())diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.pyindex 06a33b687..815de204b 100644--- a/test/environments/test_multi_step_env.py+++ b/test/environments/test_multi_step_env.py@@ -20,7 +20,6 @@ import pytestfrom camel.environments import (Action,Observation,- StepResult,)from camel.environments.multi_step import MultiStepEnv@@ -65,7 +64,7 @@ async def test_multi_step_env_lifecycle():mock_extractor = MagicMock()mock_extractor.setup = AsyncMock()mock_extractor.cleanup = AsyncMock()- mock_extractor.extract = MagicMock(return_value="extracted_answer")+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)await env.setup()@@ -77,25 +76,23 @@ async def test_multi_step_env_lifecycle():assert observation.question == "Step 0"# Test first step- action1 = Action(index=0, llm_response="Response 1")+ action1 = Action(llm_response="Response 1")# FIXME: index set to 0 temporarily until we have batched MultiStep- result1 = await env.step(action1)- assert isinstance(result1, StepResult)- assert result1.reward == 1.0- assert result1.done is False- assert result1.observation.question == "Step 1"+ obs, reward, done, info = await env.step(action1)+ assert reward == 1.0+ assert done is False+ assert obs.question == "Step 1"assert env._current_step == 1assert env._state["step"] == 1mock_extractor.extract.assert_called_once_with("Response 1")# Test second step- action2 = Action(index=0, llm_response="Response 2")+ action2 = Action(llm_response="Response 2")# FIXME: index set to 0 temporarily until we have batched MultiStep- result2 = await env.step(action2)- assert isinstance(result2, StepResult)- assert result2.reward == 1.0- assert result2.done is True- assert result2.observation.question == "Episode ended"+ obs2, reward2, done2, info = await env.step(action2)+ assert reward2 == 1.0+ assert done2 is True+ assert obs2.question == "Episode ended"assert env._current_step == 2assert env._state["step"] == 2assert mock_extractor.extract.call_count == 2@@ -147,7 +144,7 @@ async def test_multi_step_env_error_handling():# **Test Case 4: Extractor failure**# Configure extractor to raise an exception- mock_extractor.extract = MagicMock(+ mock_extractor.extract = AsyncMock(side_effect=Exception("Extractor error"))await env.reset()@@ -157,17 +154,13 @@ async def test_multi_step_env_error_handling():# **Test Case 5: Maximum steps reached**# Create a new environment with max_steps=1# Reset extractor for Test Case 5- mock_extractor.extract = MagicMock(return_value="extracted_answer")+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")env_max_steps = MockMultiStepEnv(extractor=mock_extractor, max_steps=1)await env_max_steps.setup()await env_max_steps.reset()# First step should end the episode due to max_steps- result1 = await env_max_steps.step(action)- assert (- result1.done is True- ), "Episode should be done after reaching max_steps"+ obs, reward, done, info = await env_max_steps.step(action)+ assert done is True, "Episode should be done after reaching max_steps"assert env_max_steps._current_step == 1, "Current step should be 1"assert env_max_steps.max_steps == 1, "Max steps should be 1"- assert (- result1.observation.question == "Episode ended"- ), "Should reach terminal observation"+ assert obs.question == "Episode ended", "Should reach terminal observation"diff --git a/test/environments/test_tictactoe_env.py b/test/environments/test_tictactoe_env.pynew file mode 100644index 000000000..98ea152bb--- /dev/null+++ b/test/environments/test_tictactoe_env.py@@ -0,0 +1,347 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+++import pytest++from camel.environments.models import Action+from camel.environments.tic_tac_toe import Opponent, TicTacToeEnv++# --- Setup and Initialization Tests ---+++async def test_setup_and_reset():+ """Test setup idempotency and reset functionality."""+ env = TicTacToeEnv()+ await env.setup()+ assert env._is_setup is True+ await env.setup() # Idempotent check+ assert env._is_setup is True+ observation = await env.reset()+ state = env._state+ assert state["board"] == [" " for _ in range(9)]+ assert state["game_over"] is False+ assert state["winner"] is None+ assert state["last_move_illegal"] is False+ assert "You are playing Tic Tac Toe" in observation.question+ assert "Choose a number between 1 and 9" in observation.question+ assert env._current_step == 0+ await env.close()+++# --- Step Tests: Moves ---+ "input_move, expected_behavior",+ [+ ("5", "valid"),+ ("10", "invalid_move"),+ ("-1", "invalid_input"),+ ],+)+async def test_moves(input_move, expected_behavior):+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ action = Action(llm_response=f"<Action>{input_move}</Action>")++ obs, reward, done, info = await env.step(action)+ state = env._state++ if (+ expected_behavior == "invalid_input"+ or expected_behavior == "invalid_move"+ ):+ assert state["board"] == [" " for _ in range(9)]+ assert state["last_move_illegal"] is True+ assert reward == 0.0+ if expected_behavior == "invalid_input":+ assert state["extraction_error"] is not None+ else:+ move_index = int(input_move) - 1+ assert state["board"][move_index] == "X"+ assert state["board"].count("X") == 1+ assert state["board"].count("O") == 1+ assert state["last_move_illegal"] is False++ expected = TicTacToeEnv.evaluate_position_for_x(+ state["board"], is_x_turn=True+ )+ assert reward == expected++ assert done is False+ await env.close()+++async def test_move_occupied():+ """Test moving to an occupied position."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ await env.step(Action(llm_response="<Action>1</Action>"))+ obs, reward, done, info = await env.step(+ Action(llm_response="<Action>1</Action>")+ )+ state = env._state+ assert state["last_move_illegal"] is True+ assert state["board"][0] == "X"+ assert reward == 0.0+ assert done is False+ assert "Your last move was illegal" in obs.question+ await env.close()+++# --- Step Tests: Invalid Inputs ---+++ "invalid_input", ["<Action>abc</Action>", "5", "<Action></Action>"]+)+async def test_invalid_inputs(invalid_input):+ """Test invalid action inputs."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ action = Action(llm_response=invalid_input)++ obs, reward, done, info = await env.step(action)+ state = env._state++ assert state["last_move_illegal"] is True+ assert state["extraction_error"] is not None or invalid_input == "5"+ assert reward == 0.0+ assert done is False++ await env.close()+++# --- Game End Tests ---+++async def test_agent_wins():+ """Test agent winning."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ env._state["board"] = ["X", "X", " ", "O", "O", " ", " ", " ", " "]+ obs, reward, done, info = await env.step(+ Action(llm_response="<Action>3</Action>")+ )+ state = env._state+ assert state["board"] == ["X", "X", "X", "O", "O", " ", " ", " ", " "]+ assert state["game_over"] is True+ assert state["winner"] == "X"+ assert reward == 1.0+ assert done is True+ assert "Congratulations, you won!" in obs.question+ await env.close()+++async def test_opponent_wins():+ """Test opponent winning after agent's move."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ env._state["board"] = ["O", " ", "O", "X", "X", " ", " ", " ", " "]+ obs, reward, done, info = await env.step(+ Action(llm_response="<Action>7</Action>")+ )+ state = env._state+ # The opponent should play optimally to win.+ assert state["board"][1] == "O" # Opponent places 'O' to win+ assert state["game_over"] is True+ assert state["winner"] == "O"+ assert reward == 0.0+ assert done is True+ assert "Sorry, you lost!" in obs.question+ await env.close()+++async def test_draw():+ """Test a draw."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ env._state["board"] = ["X", "O", "X", "O", "O", "X", "X", " ", "O"]+ obs, reward, done, info = await env.step(+ Action(llm_response="<Action>8</Action>")+ )+ state = env._state+ assert state["board"] == ["X", "O", "X", "O", "O", "X", "X", "X", "O"]+ assert state["game_over"] is True+ assert state["winner"] == "draw"+ assert reward == 0.5+ assert done is True+ assert "It's a draw!" in obs.question+ await env.close()+++# --- Helper Method Tests ---+++def test_helper_methods():+ """Test render_board, check_winner, and available_moves."""+ env = TicTacToeEnv()+ # Test render_board+ board = ["X", "O", " ", " ", "X", " ", " ", " ", "O"]+ expected_board = "X | O | 3\n---------\n4 | X | 6\n---------\n7 | 8 | O"+ assert env.render_board(board) == expected_board+ # Test check_winner+ assert (+ TicTacToeEnv.check_winner(+ ["X", "X", "X", "O", "O", " ", " ", " ", " "]+ )+ == "X"+ )+ assert (+ TicTacToeEnv.check_winner(+ ["O", " ", " ", "O", " ", " ", "O", " ", " "]+ )+ == "O"+ )+ assert (+ TicTacToeEnv.check_winner(+ ["X", "O", "X", "X", "O", "O", "O", "X", "X"]+ )+ == "draw"+ )+ assert (+ TicTacToeEnv.check_winner(+ ["X", "O", " ", " ", " ", " ", " ", " ", " "]+ )+ is None+ )+ # Test available_moves+ assert TicTacToeEnv.available_moves(board) == [2, 3, 5, 6, 7]+++# --- Opponent Class Tests ---+++def test_opponent_move_and_minimax():+ """Test opponent move and minimax algorithm directly+ from Opponent class."""+ opponent = Opponent(play_style="optimal")+ board = ["X", "X", " ", " ", "O", " ", " ", " ", " "]+ # The optimal move should be index 2 (blocking X's win).+ assert opponent.select_move(board) == 2+ score, move = opponent.minimax(board, is_maximizing=False)+ assert score == -1+ assert move == 2+++def test_opponent_random_move():+ """Test that a random opponent move is within the available moves."""+ opponent = Opponent(play_style="random")+ board = ["X", "O", " ", " ", "X", " ", " ", " ", " "]+ available = TicTacToeEnv.available_moves(board)+ move = opponent.select_move(board)+ assert move in available+++def test_opponent_no_moves():+ """Test that when no moves are available, the opponent returns None."""+ opponent = Opponent(play_style="optimal")+ board = ["X", "O", "X", "X", "O", "O", "O", "X", "X"]+ move = opponent.select_move(board)+ assert move is None+++def test_opponent_optimal_winning_move():+ """Test that the optimal opponent identifies and makes a winning move,+ if available."""+ opponent = Opponent(play_style="optimal")+ # Set up a board where O can win immediately (row 0: two O's and an empty).+ board = ["O", "O", " ", "X", "X", " ", " ", " ", " "]+ move = opponent.select_move(board)+ # The winning move is index 2.+ assert move == 2+++def test_minimax_draw_scenario():+ """Test that minimax returns a draw score for a board leading to a draw."""+ opponent = Opponent(play_style="optimal")+ # A nearly complete board with one move left that results in a draw.+ board = ["X", "O", "X", "X", "O", "O", "O", "X", " "]+ score, move = opponent.minimax(board, is_maximizing=True)+ # Only one move is available; playing it leads to a draw.+ assert score == 0+ # The only available move should be index 8.+ assert move == 8+++# --- Error Handling Tests ---+++async def test_step_errors():+ """Test errors for stepping without setup, after done, or without reset."""+ # Without setup+ env = TicTacToeEnv()+ with pytest.raises(RuntimeError, match="Environment not set up"):+ await env.step(Action(llm_response="<Action>5</Action>"))++ env = TicTacToeEnv()+ await env.setup()+ await env.reset()+ env._episode_ended = True+ with pytest.raises(RuntimeError, match="Episode has ended"):+ await env.step(Action(llm_response="<Action>5</Action>"))+ await env.close()++ env = TicTacToeEnv()+ await env.setup()+ env._last_observation = None # Simulate no reset+ with pytest.raises(RuntimeError, match="No current observation"):+ await env.step(Action(llm_response="<Action>5</Action>"))+ await env.close()+++# --- Full test ---+async def test_full_game():+ """Test a full game sequence with intermediate reward shaping."""+ env = TicTacToeEnv()+ await env.setup()+ await env.reset()++ moves = ["1", "2", "4"]+ for move in moves:+ obs, reward, done, info = await env.step(+ Action(llm_response=f"<Action>{move}</Action>")+ )+ if done:+ winner = env._state["winner"]+ if winner == "X":+ assert reward == 1.0+ elif winner == "O":+ assert reward == 0.0+ elif winner == "draw":+ assert reward == 0.5+ else:+ # NEW: Validate reward matches optimal X evaluation+ expected = TicTacToeEnv.evaluate_position_for_x(+ env._state["board"], is_x_turn=True+ )+ assert reward == expected+ assert 0.0 <= reward <= 1.0+ assert done is False++ await env.close()