Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: implement Tic Tac Toe as `MultiStepEnv` (#1914)

hallerite5069b893896f
diff --git a/camel/environments/__init__.py b/camel/environments/__init__.py
index c580dd577..cfd925736 100644
--- a/camel/environments/__init__.py
+++ b/camel/environments/__init__.py
@@ -14,6 +14,7 @@
from .models import Action, Environment, Observation, StepResult
from .multi_step import MultiStepEnv
from .single_step import SingleStepEnv
+from .tic_tac_toe import Opponent, TicTacToeEnv
__all__ = [
"Environment",
@@ -22,4 +23,6 @@ __all__ = [
"Action",
"Observation",
"StepResult",
+ "TicTacToeEnv",
+ "Opponent",
]
diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.py
index df0013954..e73b2af6d 100644
--- a/camel/environments/multi_step.py
+++ b/camel/environments/multi_step.py
@@ -15,11 +15,10 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Tuple
+from camel.environments.models import Action, Observation, StepResult
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
-from .models import Action, Observation, StepResult
-
logger = get_logger(__name__)
@@ -73,9 +72,8 @@ class MultiStepEnv(ABC):
logger.error(f'Failed to setup environment: {e}')
raise
- @abstractmethod
async def _setup(self) -> None:
- pass
+ return
async def close(self) -> None:
r"""Clean up and close all resources used by the environment.
@@ -100,9 +98,8 @@ class MultiStepEnv(ABC):
logger.error(f'Failed to teardown environment: {e}')
raise
- @abstractmethod
async def _close(self) -> None:
- pass
+ return
async def reset(self) -> Observation:
r"""Reset the environment to an initial state.
@@ -115,6 +112,9 @@ class MultiStepEnv(ABC):
"""
if not self._is_setup:
+ logger.warning(
+ "reset() called on un-setup environment. Setting up..."
+ )
await self.setup()
# Reset state
@@ -132,7 +132,9 @@ class MultiStepEnv(ABC):
return observation
- async def step(self, action: Action) -> StepResult:
+ async def step(
+ self, action: Action
+ ) -> Tuple[Observation, float, bool, Dict[str, Any]]:
r"""Take a step in the environment using the given action.
This method updates the environment state based on the LLM's response,
@@ -157,7 +159,7 @@ class MultiStepEnv(ABC):
rewards_dict={},
done=True,
info={"reason": "max_steps_reached"},
- )
+ ).as_tuple()
if not self._is_setup:
raise RuntimeError("Environment not set up. Call setup() first.")
@@ -196,13 +198,13 @@ class MultiStepEnv(ABC):
rewards_dict=rewards_dict,
done=done,
info={
- "extraction_result": self.extractor.extract(
+ "extraction_result": await self.extractor.extract(
action.llm_response
),
"step": self._current_step,
"state": self._state, # Updated state
},
- )
+ ).as_tuple()
@abstractmethod
def _get_initial_state(self) -> Dict[str, Any]:
diff --git a/camel/environments/tic_tac_toe.py b/camel/environments/tic_tac_toe.py
new file mode 100644
index 000000000..be6f17896
--- /dev/null
+++ b/camel/environments/tic_tac_toe.py
@@ -0,0 +1,518 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import math
+import random
+import re
+from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple
+
+from camel.environments.models import Action, Observation
+from camel.environments.multi_step import MultiStepEnv
+from camel.extractors import BaseExtractor, BaseExtractorStrategy
+
+
+class MoveExtractor(BaseExtractorStrategy):
+ r"""A strategy for extracting Tic Tac Toe actions from text."""
+
+ async def extract(self, text: str) -> Optional[str]:
+ r"""Extract a valid Tic Tac Toe move from text.
+
+ Looks for a pattern '<Action> n' where n is a digit between 1 and 9.
+
+ Args:
+ text (str): The text to extract the action from.
+
+ Returns:
+ Optional[str]: The extracted move as a string, or None if no valid
+ move is found.
+ """
+ match = re.search(r"<Action>\s*(\d+)", text)
+ if match:
+ move = match.group(1)
+ # Validate that the move is in range 1-9
+ if move.isdigit() and 1 <= int(move) <= 9:
+ return move
+ return None
+
+
+class Opponent:
+ r"""AI opponent for the Tic Tac Toe game.
+
+ This class implements different playing strategies for the AI opponent,
+ including an optimal strategy using the minimax algorithm with alpha-beta
+ pruning, and a random strategy.
+ """
+
+ def __init__(
+ self, play_style: Literal["optimal", "random"] = "optimal"
+ ) -> None:
+ r"""Initialize the opponent with a specific play style.
+
+ Args:
+ play_style (Literal["optimal", "random"]): The strategy to use,
+ either "optimal" or "random". (default: :obj:`"optimal"`)
+ """
+ self.play_style = play_style
+
+ def select_move(self, board: List[str]) -> Optional[int]:
+ r"""Select a move based on the opponent's play style.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+
+ Returns:
+ Optional[int]: The index of the selected move, or None if no move
+ is available.
+ """
+ if self.play_style == "optimal":
+ return self.get_optimal_move(board)
+ elif self.play_style == "random":
+ moves = TicTacToeEnv.available_moves(board)
+ if not moves:
+ return None # Consistent with optimal strategy
+ return random.choice(moves)
+
+ def get_optimal_move(self, board: List[str]) -> Optional[int]:
+ r"""Get the optimal move using the minimax algorithm.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+
+ Returns:
+ Optional[int]: The index of the optimal move, or None if no move
+ is available.
+ """
+ _, move = self.minimax(board, is_maximizing=True)
+ return move
+
+ def minimax(
+ self,
+ board: List[str],
+ is_maximizing: bool,
+ depth: int = 0,
+ alpha: float = -math.inf,
+ beta: float = math.inf,
+ ) -> Tuple[float, Optional[int]]:
+ r"""Minimax algorithm with alpha-beta pruning for optimal move
+ selection.
+
+ Recursively evaluates all possible moves to find the best one.
+ Uses alpha-beta pruning to reduce the search space.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+ is_maximizing (bool): True if maximizing player (O), False if
+ minimizing (X).
+ depth (int): Current depth in the search tree. (default: :obj:`0`)
+ alpha (float): Alpha value for pruning. (default: :obj:`-math.inf`)
+ beta (float): Beta value for pruning. (default: :obj:`math.inf`)
+
+ Returns:
+ Tuple[float, Optional[int]]: A tuple containing:
+ - float: The score of the best move (1 for O win, -1 for X
+ win, 0 for draw)
+ - Optional[int]: The index of the best move, or None if
+ terminal state
+ """
+ winner = TicTacToeEnv.check_winner(board)
+ if winner == "O":
+ return (1, None)
+ elif winner == "X":
+ return (-1, None)
+ elif winner == "draw":
+ return (0, None)
+
+ moves = TicTacToeEnv.available_moves(board)
+ # Add depth limit to prevent stack overflow (9 is max depth for
+ # tic-tac-toe)
+ if depth >= 9:
+ # Evaluate current position
+ return (0, None)
+
+ if is_maximizing:
+ best_score = -math.inf
+ best_move = None
+ for move in moves:
+ board[move] = "O"
+ score, _ = self.minimax(
+ board,
+ is_maximizing=False,
+ depth=depth + 1,
+ alpha=alpha,
+ beta=beta,
+ )
+ board[move] = " "
+ if score > best_score:
+ best_score = score
+ best_move = move
+ alpha = max(alpha, best_score)
+ if beta <= alpha:
+ break # Beta cutoff
+ return best_score, best_move
+ else:
+ best_score = math.inf
+ best_move = None
+ for move in moves:
+ board[move] = "X"
+ score, _ = self.minimax(
+ board,
+ is_maximizing=True,
+ depth=depth + 1,
+ alpha=alpha,
+ beta=beta,
+ )
+ board[move] = " "
+ if score < best_score:
+ best_score = score
+ best_move = move
+ beta = min(beta, best_score)
+ if beta <= alpha:
+ break # Alpha cutoff
+ return best_score, best_move
+
+
+class TicTacToeEnv(MultiStepEnv):
+ r"""A Tic Tac Toe environment for reinforcement learning with LLMs.
+
+ This environment implements a standard Tic Tac Toe game where the LLM agent
+ plays as 'X' against an AI opponent that plays as 'O'. The opponent can use
+ either an optimal strategy (minimax with alpha-beta pruning) or a random
+ strategy.
+ """
+
+ WIN_COMBINATIONS: ClassVar = [
+ (0, 1, 2), # Top row
+ (3, 4, 5), # Middle row
+ (6, 7, 8), # Bottom row
+ (0, 3, 6), # Left column
+ (1, 4, 7), # Middle column
+ (2, 5, 8), # Right column
+ (0, 4, 8), # Diagonal from top-left
+ (2, 4, 6), # Diagonal from top-right
+ ]
+
+ def __init__(
+ self,
+ extractor: Optional[BaseExtractor] = None,
+ max_steps: Optional[int] = None,
+ play_style: Literal["optimal", "random"] = "optimal",
+ **kwargs,
+ ) -> None:
+ r"""Initialize the Tic Tac Toe environment.
+
+ Args:
+ extractor (Optional[BaseExtractor]): Extractor to process LLM
+ responses. If None, a default extractor with
+ MoveExtractor will be used. (default: :obj:`None`)
+ max_steps (Optional[int]): Maximum steps per episode.
+ (default: :obj:`None`)
+ play_style (Literal["optimal", "random"]): The strategy for the
+ opponent to use, either "optimal" or "random". (default:
+ :obj:`"optimal"`)
+ **kwargs: Additional environment parameters.
+ """
+ if extractor is None:
+ extractor = BaseExtractor(pipeline=[[MoveExtractor()]])
+ super().__init__(extractor, max_steps, **kwargs)
+ self.opponent = Opponent(play_style=play_style)
+
+ def _get_initial_state(self) -> Dict[str, Any]:
+ r"""Get the initial state of the environment.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing the initial state with an
+ empty board, game status flags, and move history.
+ """
+ # State includes the board (9 cells), game_over flag, and winner info.
+ return {
+ "board": [" " for _ in range(9)],
+ "game_over": False,
+ "winner": None,
+ "last_move_illegal": False,
+ "last_move": None,
+ }
+
+ async def _update_state(self, action: Action) -> None:
+ r"""Update the environment state based on the agent's action.
+
+ This method processes the agent's move, updates the board, checks for
+ a winner, and if the game is not over, makes a move for the opponent.
+
+ Args:
+ action (Action): The action containing the LLM's response with the
+ chosen move.
+
+ Returns:
+ None
+ """
+ board = self._state["board"]
+
+ # Attempt to parse the agent's chosen move
+ extraction_result = await self.extractor.extract(action.llm_response)
+ if not extraction_result:
+ # Handle extraction failure gracefully
+ self._state["last_move_illegal"] = True
+ self._state["last_move"] = None
+ self._state["extraction_error"] = "Could not extract a valid move"
+ return
+
+ try:
+ move = int(extraction_result)
+ self._state["last_move"] = move
+ self._state["extraction_error"] = None
+ except ValueError:
+ # Handle invalid move format gracefully
+ self._state["last_move_illegal"] = True
+ self._state["last_move"] = extraction_result
+ self._state["extraction_error"] = (
+ f"'{extraction_result}' is not a valid number"
+ )
+ return
+
+ # Convert 1-indexed move to 0-indexed board position.
+ index = move - 1
+ if index < 0 or index > 8 or board[index] != " ":
+ self._state["last_move_illegal"] = True
+ self._state["extraction_error"] = (
+ f"Position {move} is not a valid or available move"
+ )
+ return
+
+ # Reset the flag
+ self._state["last_move_illegal"] = False
+
+ # Agent (X) makes the move.
+ board[index] = "X"
+
+ # Check if agent wins (or draw) right after its move.
+ winner = self.check_winner(board)
+ if winner is not None:
+ self._state["game_over"] = True
+ self._state["winner"] = winner
+ return
+
+ # Opponent (O) plays using the opponent class.
+ opponent_move = self.opponent.select_move(board)
+ if opponent_move is not None:
+ board[opponent_move] = "O"
+
+ # Check if the game ended after opponent's move.
+ winner = self.check_winner(board)
+ if winner is not None:
+ self._state["game_over"] = True
+ self._state["winner"] = winner
+
+ def _get_next_observation(self) -> Observation:
+ r"""Get the next observation based on the current state.
+
+ This method generates a text observation describing the current state
+ of the game board and prompting the agent to make a move.
+
+ Returns:
+ Observation: An Observation object containing the game state
+ description.
+ """
+ board = self._state["board"]
+ if self._state["last_move_illegal"]:
+ obs = (
+ "You are playing Tic Tac Toe with standard rules.\n"
+ "You are the player with X.\n"
+ "Your last move was illegal.\n"
+ f"You chose the move {self._state['last_move']}."
+ "Choose another number between 1 and 9 to place an X.\n"
+ "The field must still be available.\n"
+ "This is the current state of the board:\n"
+ f"{self.render_board(board)}\n"
+ "Each number that you can see is still an empty field "
+ "that you can place your 'X' in. Please end your response "
+ "with <Action> [a number from 1 to 9]"
+ )
+ else:
+ obs = (
+ "You are playing Tic Tac Toe with standard rules.\n"
+ "You are the player with X.\n"
+ "Choose a number between 1 and 9 to place an X.\n"
+ "This is the current state of the board:\n"
+ f"{self.render_board(board)}\n"
+ "Each number that you can see is still an empty field "
+ "that you can place your 'X' in. Please end your response "
+ "with <Action> [a number from 1 to 9]"
+ )
+
+ return Observation(question=obs, context={}, metadata={})
+
+ def _get_terminal_observation(self) -> Observation:
+ r"""Get the final observation when the game is over.
+
+ This method generates a text observation describing the final state
+ of the game board and the game result (win, loss, or draw).
+
+ Returns:
+ Observation: An Observation object containing the final game state
+ description.
+ """
+ board = self._state["board"]
+ result_message = ""
+ if self._state["winner"] == "X":
+ result_message = "Congratulations, you won!"
+ elif self._state["winner"] == "O":
+ result_message = "Sorry, you lost!"
+ else:
+ result_message = "It's a draw!"
+
+ obs = f"{self.render_board(board)}\nGame Over. {result_message}"
+
+ return Observation(question=obs, context={}, metadata={})
+
+ async def compute_reward(self) -> Tuple[float, Dict[str, float]]:
+ r"""Compute the reward for the current state.
+
+ Returns:
+ Tuple[float, Dict[str, float]]: A tuple containing the total
+ reward and a dictionary of reward components:
+ - 1.0 for a win
+ - 0.0 for a loss or illegal move
+ - 0.5 for a draw
+ - For ongoing games, returns an evaluation of the position
+ """
+ # Simple reward: 1 for win, 0 for loss, 0.5 for draw or ongoing.
+ if self._state["game_over"]:
+ if self._state["winner"] == "X":
+ return 1.0, {"win": 1.0}
+ elif self._state["winner"] == "O":
+ return 0.0, {"loss": 0.0}
+ else:
+ return 0.5, {"draw": 0.5}
+
+ elif self._state["last_move_illegal"]:
+ return 0.0, {"illegal_move": 0.0}
+
+ else:
+ board = self._state["board"]
+ value = TicTacToeEnv.evaluate_position_for_x(board, is_x_turn=True)
+ return value, {"x_non_loss_value": value}
+
+ @staticmethod
+ def evaluate_position_for_x(
+ board: List[str], is_x_turn: bool, depth: int = 0, max_depth: int = 10
+ ) -> float:
+ r"""Evaluate the current board position from X's perspective.
+
+ Uses minimax to determine the value of the position.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+ is_x_turn (bool): True if it's X's turn to move, False otherwise.
+
+ Returns:
+ float: A float value representing the position evaluation:
+ - 1.0 if X has a winning position
+ - 0.0 if O has a winning position
+ - 0.5 for a draw
+ - For ongoing positions, returns the expected outcome with
+ perfect play
+ """
+ winner = TicTacToeEnv.check_winner(board)
+ if winner == "X":
+ return 1.0 # X wins
+ elif winner == "O":
+ return 0.0 # X loses
+ elif winner == "draw":
+ return 0.5 # draw
+
+ # Add depth limit to prevent potential stack overflow
+ if depth >= max_depth:
+ return 0.5 # Return draw evaluation at max depth
+
+ moves = TicTacToeEnv.available_moves(board)
+ values = []
+ # Create a copy of the board to avoid side effects
+ for move in moves:
+ board_copy = board.copy()
+ board_copy[move] = "X" if is_x_turn else "O"
+ value = TicTacToeEnv.evaluate_position_for_x(
+ board_copy, not is_x_turn, depth + 1, max_depth
+ )
+ values.append(value)
+
+ return max(values) if is_x_turn else min(values)
+
+ def _is_done(self) -> bool:
+ r"""Check if the episode is done.
+
+ Returns:
+ True if the game is over, False otherwise.
+ """
+ return self._state["game_over"]
+
+ @staticmethod
+ def available_moves(board: List[str]) -> List[int]:
+ r"""Get all available moves on the board.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+
+ Returns:
+ List[int]: A list of indices representing empty cells on the board.
+ """
+ # Return list of indices that are free.
+ return [i for i, cell in enumerate(board) if cell == " "]
+
+ @staticmethod
+ def check_winner(board: List[str]) -> Optional[Literal["X", "O", "draw"]]:
+ r"""Check if there is a winner or a draw on the board.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+
+ Returns:
+ Optional[Literal["X", "O", "draw"]]: "X" if X has won, "O" if O
+ has won, "draw" if the game is a draw, or None if the game is
+ still ongoing.
+ """
+ # Check all win combinations.
+ for a, b, c in TicTacToeEnv.WIN_COMBINATIONS:
+ if board[a] != " " and board[a] == board[b] == board[c]:
+ return board[a]
+ # Check for draw.
+ if all(cell != " " for cell in board):
+ return "draw"
+ return None
+
+ def render_board(self, board: List[str]) -> str:
+ r"""Render the board as a string for display.
+
+ Args:
+ board (List[str]): The current game board as a list of strings.
+
+ Returns:
+ str: A formatted string representation of the board.
+ """
+
+ # Create a nice formatted board.
+ def cell_value(i: int) -> str:
+ r"""Get the display value for a cell.
+
+ Args:
+ i (int): The index of the cell.
+
+ Returns:
+ str: The cell content ("X" or "O") or the cell number if empty.
+ """
+ return board[i] if board[i] != " " else str(i + 1)
+
+ rows = []
+ for i in range(0, 9, 3):
+ row = " | ".join(cell_value(j) for j in range(i, i + 3))
+ rows.append(row)
+ return "\n---------\n".join(rows)
diff --git a/docs/cookbooks/loong/multi_step_rl.ipynb b/docs/cookbooks/loong/multi_step_rl.ipynb
new file mode 100644
index 000000000..58cd47c5d
--- /dev/null
+++ b/docs/cookbooks/loong/multi_step_rl.ipynb
@@ -0,0 +1,206 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/student/true-gnosis/Code/camel/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Initial Observation:\n",
+ "\n",
+ "You are playing Tic Tac Toe with standard rules.\n",
+ "You are the player with X.\n",
+ "Choose a number between 1 and 9 to place an X.\n",
+ "This is the current state of the board:\n",
+ "1 | 2 | 3\n",
+ "---------\n",
+ "4 | 5 | 6\n",
+ "---------\n",
+ "7 | 8 | 9\n",
+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n"
+ ]
+ }
+ ],
+ "source": [
+ "import asyncio\n",
+ "from camel.environments.models import Action\n",
+ "from camel.environments.tic_tac_toe import TicTacToeEnv, Opponent # Adjust if needed\n",
+ "\n",
+ "# Initialize and set up the environment\n",
+ "env = TicTacToeEnv(opponent=Opponent(play_style=\"random\"))\n",
+ "await env.setup()\n",
+ "\n",
+ "# Reset environment and get initial observation\n",
+ "observation = await env.reset()\n",
+ "print(\"Initial Observation:\\n\")\n",
+ "print(observation.question)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from getpass import getpass\n",
+ "\n",
+ "openai_api_key = getpass('Enter your API key: ')\n",
+ "os.environ[\"OPENAI_API_KEY\"] = openai_api_key"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from camel.models import ModelFactory\n",
+ "from camel.types import ModelPlatformType, ModelType\n",
+ "from camel.configs import ChatGPTConfig\n",
+ "from camel.agents import ChatAgent\n",
+ "\n",
+ "model = ModelFactory.create(\n",
+ " model_platform=ModelPlatformType.OPENAI,\n",
+ " model_type=ModelType.GPT_4O_MINI,\n",
+ " model_config_dict=ChatGPTConfig().as_dict(),\n",
+ ")\n",
+ "\n",
+ "agent = ChatAgent(model=model, system_message=\"PLEASE RESPOND WITH <Action>\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Agent Move: index=0 llm_response=\"I'll place my X in position 5. \\n\\n<Action> 5\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 47, 647354, tzinfo=datetime.timezone.utc)\n",
+ "Observation:\n",
+ "You are playing Tic Tac Toe with standard rules.\n",
+ "You are the player with X.\n",
+ "Choose a number between 1 and 9 to place an X.\n",
+ "This is the current state of the board:\n",
+ "O | 2 | 3\n",
+ "---------\n",
+ "4 | X | 6\n",
+ "---------\n",
+ "7 | 8 | 9\n",
+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",
+ "Reward: 0.5\n",
+ "Done: False\n",
+ "Info: {'extraction_result': '5', 'step': 1, 'state': {'board': ['O', ' ', ' ', ' ', 'X', ' ', ' ', ' ', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 5}, 'rewards_dict': {'x_non_loss_value': 0.5}}\n",
+ "\n",
+ "Agent Move: index=0 llm_response=\"I'll place my X in position 2.\\n\\n<Action> 2\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 48, 369680, tzinfo=datetime.timezone.utc)\n",
+ "Observation:\n",
+ "You are playing Tic Tac Toe with standard rules.\n",
+ "You are the player with X.\n",
+ "Choose a number between 1 and 9 to place an X.\n",
+ "This is the current state of the board:\n",
+ "O | X | 3\n",
+ "---------\n",
+ "4 | X | 6\n",
+ "---------\n",
+ "7 | O | 9\n",
+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",
+ "Reward: 0.5\n",
+ "Done: False\n",
+ "Info: {'extraction_result': '2', 'step': 2, 'state': {'board': ['O', 'X', ' ', ' ', 'X', ' ', ' ', 'O', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 2}, 'rewards_dict': {'x_non_loss_value': 0.5}}\n",
+ "\n",
+ "Agent Move: index=0 llm_response=\"I'll place my X in position 3.\\n\\n<Action> 3\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 49, 25743, tzinfo=datetime.timezone.utc)\n",
+ "Observation:\n",
+ "You are playing Tic Tac Toe with standard rules.\n",
+ "You are the player with X.\n",
+ "Choose a number between 1 and 9 to place an X.\n",
+ "This is the current state of the board:\n",
+ "O | X | X\n",
+ "---------\n",
+ "4 | X | 6\n",
+ "---------\n",
+ "O | O | 9\n",
+ "Each number that you can see is still an empty fieldthat you can place your 'X' in. Please end your response with<Action> [a number from 1 to 9]\n",
+ "Reward: 0.0\n",
+ "Done: False\n",
+ "Info: {'extraction_result': '3', 'step': 3, 'state': {'board': ['O', 'X', 'X', ' ', 'X', ' ', 'O', 'O', ' '], 'game_over': False, 'winner': None, 'last_move_illegal': False, 'last_move': 3}, 'rewards_dict': {'x_non_loss_value': 0.0}}\n",
+ "\n",
+ "Agent Move: index=0 llm_response=\"I'll place my X in position 6.\\n\\n<Action> 6\" metadata={} timestamp=datetime.datetime(2025, 4, 7, 23, 44, 49, 588802, tzinfo=datetime.timezone.utc)\n",
+ "Observation:\n",
+ "O | X | X\n",
+ "---------\n",
+ "O | X | X\n",
+ "---------\n",
+ "O | O | 9\n",
+ "Game Over. Sorry, you lost!\n",
+ "Reward: 0.0\n",
+ "Done: True\n",
+ "Info: {'extraction_result': '6', 'step': 4, 'state': {'board': ['O', 'X', 'X', 'O', 'X', 'X', 'O', 'O', ' '], 'game_over': True, 'winner': 'O', 'last_move_illegal': False, 'last_move': 6}, 'rewards_dict': {'loss': 0.0}}\n"
+ ]
+ }
+ ],
+ "source": [
+ "while not env.is_done():\n",
+ "\n",
+ " llm_response = agent.step(observation.question).msgs[0].content\n",
+ " #agent.reset()\n",
+ "\n",
+ " action = Action(llm_response=llm_response)\n",
+ " result = await env.step(action)\n",
+ "\n",
+ " observation, reward, done, info = result\n",
+ "\n",
+ " print(\"\\nAgent Move:\", action)\n",
+ " print(\"Observation:\")\n",
+ " print(observation.question)\n",
+ " print(\"Reward:\", reward)\n",
+ " print(\"Done:\", done)\n",
+ " print(\"Info:\", info)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Close the environment\n",
+ "await env.close()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/environments/tic_tac_toe.py b/examples/environments/tic_tac_toe.py
new file mode 100644
index 000000000..db7105226
--- /dev/null
+++ b/examples/environments/tic_tac_toe.py
@@ -0,0 +1,57 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+import random
+
+from camel.environments import TicTacToeEnv
+from camel.environments.models import Action
+
+
+async def run_game():
+ env = TicTacToeEnv()
+ # Initialize and set up the environment
+ await env.setup()
+
+ # Reset environment and get initial observation
+ observation = await env.reset()
+ print("Initial Observation:\n")
+ print(observation.question)
+
+ while not env.is_done():
+ available_moves = [
+ move + 1
+ for move in TicTacToeEnv.available_moves(env._state["board"])
+ ]
+
+ action = Action(
+ llm_response=f"<Action> {random.choice(available_moves)}"
+ )
+ result = await env.step(action)
+
+ observation, reward, done, info = result
+
+ print("\nAgent Move:", action)
+ print("Observation:")
+ print(observation.question)
+ print("Reward:", reward)
+ print("Done:", done)
+ print("Info:", info)
+
+ print("\n___________\n\nBoard at the end:\n")
+ print(env.render_board(env._state["board"]), "\n")
+ await env._close()
+
+
+if __name__ == "__main__":
+ asyncio.run(run_game())
diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.py
index 06a33b687..815de204b 100644
--- a/test/environments/test_multi_step_env.py
+++ b/test/environments/test_multi_step_env.py
@@ -20,7 +20,6 @@ import pytest
from camel.environments import (
Action,
Observation,
- StepResult,
)
from camel.environments.multi_step import MultiStepEnv
@@ -65,7 +64,7 @@ async def test_multi_step_env_lifecycle():
mock_extractor = MagicMock()
mock_extractor.setup = AsyncMock()
mock_extractor.cleanup = AsyncMock()
- mock_extractor.extract = MagicMock(return_value="extracted_answer")
+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")
env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
await env.setup()
@@ -77,25 +76,23 @@ async def test_multi_step_env_lifecycle():
assert observation.question == "Step 0"
# Test first step
- action1 = Action(index=0, llm_response="Response 1")
+ action1 = Action(llm_response="Response 1")
# FIXME: index set to 0 temporarily until we have batched MultiStep
- result1 = await env.step(action1)
- assert isinstance(result1, StepResult)
- assert result1.reward == 1.0
- assert result1.done is False
- assert result1.observation.question == "Step 1"
+ obs, reward, done, info = await env.step(action1)
+ assert reward == 1.0
+ assert done is False
+ assert obs.question == "Step 1"
assert env._current_step == 1
assert env._state["step"] == 1
mock_extractor.extract.assert_called_once_with("Response 1")
# Test second step
- action2 = Action(index=0, llm_response="Response 2")
+ action2 = Action(llm_response="Response 2")
# FIXME: index set to 0 temporarily until we have batched MultiStep
- result2 = await env.step(action2)
- assert isinstance(result2, StepResult)
- assert result2.reward == 1.0
- assert result2.done is True
- assert result2.observation.question == "Episode ended"
+ obs2, reward2, done2, info = await env.step(action2)
+ assert reward2 == 1.0
+ assert done2 is True
+ assert obs2.question == "Episode ended"
assert env._current_step == 2
assert env._state["step"] == 2
assert mock_extractor.extract.call_count == 2
@@ -147,7 +144,7 @@ async def test_multi_step_env_error_handling():
# **Test Case 4: Extractor failure**
# Configure extractor to raise an exception
- mock_extractor.extract = MagicMock(
+ mock_extractor.extract = AsyncMock(
side_effect=Exception("Extractor error")
)
await env.reset()
@@ -157,17 +154,13 @@ async def test_multi_step_env_error_handling():
# **Test Case 5: Maximum steps reached**
# Create a new environment with max_steps=1
# Reset extractor for Test Case 5
- mock_extractor.extract = MagicMock(return_value="extracted_answer")
+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")
env_max_steps = MockMultiStepEnv(extractor=mock_extractor, max_steps=1)
await env_max_steps.setup()
await env_max_steps.reset()
# First step should end the episode due to max_steps
- result1 = await env_max_steps.step(action)
- assert (
- result1.done is True
- ), "Episode should be done after reaching max_steps"
+ obs, reward, done, info = await env_max_steps.step(action)
+ assert done is True, "Episode should be done after reaching max_steps"
assert env_max_steps._current_step == 1, "Current step should be 1"
assert env_max_steps.max_steps == 1, "Max steps should be 1"
- assert (
- result1.observation.question == "Episode ended"
- ), "Should reach terminal observation"
+ assert obs.question == "Episode ended", "Should reach terminal observation"
diff --git a/test/environments/test_tictactoe_env.py b/test/environments/test_tictactoe_env.py
new file mode 100644
index 000000000..98ea152bb
--- /dev/null
+++ b/test/environments/test_tictactoe_env.py
@@ -0,0 +1,347 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+
+import pytest
+
+from camel.environments.models import Action
+from camel.environments.tic_tac_toe import Opponent, TicTacToeEnv
+
+# --- Setup and Initialization Tests ---
+
+
+async def test_setup_and_reset():
+ """Test setup idempotency and reset functionality."""
+ env = TicTacToeEnv()
+ await env.setup()
+ assert env._is_setup is True
+ await env.setup() # Idempotent check
+ assert env._is_setup is True
+ observation = await env.reset()
+ state = env._state
+ assert state["board"] == [" " for _ in range(9)]
+ assert state["game_over"] is False
+ assert state["winner"] is None
+ assert state["last_move_illegal"] is False
+ assert "You are playing Tic Tac Toe" in observation.question
+ assert "Choose a number between 1 and 9" in observation.question
+ assert env._current_step == 0
+ await env.close()
+
+
+# --- Step Tests: Moves ---
+ "input_move, expected_behavior",
+ [
+ ("5", "valid"),
+ ("10", "invalid_move"),
+ ("-1", "invalid_input"),
+ ],
+)
+async def test_moves(input_move, expected_behavior):
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ action = Action(llm_response=f"<Action>{input_move}</Action>")
+
+ obs, reward, done, info = await env.step(action)
+ state = env._state
+
+ if (
+ expected_behavior == "invalid_input"
+ or expected_behavior == "invalid_move"
+ ):
+ assert state["board"] == [" " for _ in range(9)]
+ assert state["last_move_illegal"] is True
+ assert reward == 0.0
+ if expected_behavior == "invalid_input":
+ assert state["extraction_error"] is not None
+ else:
+ move_index = int(input_move) - 1
+ assert state["board"][move_index] == "X"
+ assert state["board"].count("X") == 1
+ assert state["board"].count("O") == 1
+ assert state["last_move_illegal"] is False
+
+ expected = TicTacToeEnv.evaluate_position_for_x(
+ state["board"], is_x_turn=True
+ )
+ assert reward == expected
+
+ assert done is False
+ await env.close()
+
+
+async def test_move_occupied():
+ """Test moving to an occupied position."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ await env.step(Action(llm_response="<Action>1</Action>"))
+ obs, reward, done, info = await env.step(
+ Action(llm_response="<Action>1</Action>")
+ )
+ state = env._state
+ assert state["last_move_illegal"] is True
+ assert state["board"][0] == "X"
+ assert reward == 0.0
+ assert done is False
+ assert "Your last move was illegal" in obs.question
+ await env.close()
+
+
+# --- Step Tests: Invalid Inputs ---
+
+
+ "invalid_input", ["<Action>abc</Action>", "5", "<Action></Action>"]
+)
+async def test_invalid_inputs(invalid_input):
+ """Test invalid action inputs."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ action = Action(llm_response=invalid_input)
+
+ obs, reward, done, info = await env.step(action)
+ state = env._state
+
+ assert state["last_move_illegal"] is True
+ assert state["extraction_error"] is not None or invalid_input == "5"
+ assert reward == 0.0
+ assert done is False
+
+ await env.close()
+
+
+# --- Game End Tests ---
+
+
+async def test_agent_wins():
+ """Test agent winning."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ env._state["board"] = ["X", "X", " ", "O", "O", " ", " ", " ", " "]
+ obs, reward, done, info = await env.step(
+ Action(llm_response="<Action>3</Action>")
+ )
+ state = env._state
+ assert state["board"] == ["X", "X", "X", "O", "O", " ", " ", " ", " "]
+ assert state["game_over"] is True
+ assert state["winner"] == "X"
+ assert reward == 1.0
+ assert done is True
+ assert "Congratulations, you won!" in obs.question
+ await env.close()
+
+
+async def test_opponent_wins():
+ """Test opponent winning after agent's move."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ env._state["board"] = ["O", " ", "O", "X", "X", " ", " ", " ", " "]
+ obs, reward, done, info = await env.step(
+ Action(llm_response="<Action>7</Action>")
+ )
+ state = env._state
+ # The opponent should play optimally to win.
+ assert state["board"][1] == "O" # Opponent places 'O' to win
+ assert state["game_over"] is True
+ assert state["winner"] == "O"
+ assert reward == 0.0
+ assert done is True
+ assert "Sorry, you lost!" in obs.question
+ await env.close()
+
+
+async def test_draw():
+ """Test a draw."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ env._state["board"] = ["X", "O", "X", "O", "O", "X", "X", " ", "O"]
+ obs, reward, done, info = await env.step(
+ Action(llm_response="<Action>8</Action>")
+ )
+ state = env._state
+ assert state["board"] == ["X", "O", "X", "O", "O", "X", "X", "X", "O"]
+ assert state["game_over"] is True
+ assert state["winner"] == "draw"
+ assert reward == 0.5
+ assert done is True
+ assert "It's a draw!" in obs.question
+ await env.close()
+
+
+# --- Helper Method Tests ---
+
+
+def test_helper_methods():
+ """Test render_board, check_winner, and available_moves."""
+ env = TicTacToeEnv()
+ # Test render_board
+ board = ["X", "O", " ", " ", "X", " ", " ", " ", "O"]
+ expected_board = "X | O | 3\n---------\n4 | X | 6\n---------\n7 | 8 | O"
+ assert env.render_board(board) == expected_board
+ # Test check_winner
+ assert (
+ TicTacToeEnv.check_winner(
+ ["X", "X", "X", "O", "O", " ", " ", " ", " "]
+ )
+ == "X"
+ )
+ assert (
+ TicTacToeEnv.check_winner(
+ ["O", " ", " ", "O", " ", " ", "O", " ", " "]
+ )
+ == "O"
+ )
+ assert (
+ TicTacToeEnv.check_winner(
+ ["X", "O", "X", "X", "O", "O", "O", "X", "X"]
+ )
+ == "draw"
+ )
+ assert (
+ TicTacToeEnv.check_winner(
+ ["X", "O", " ", " ", " ", " ", " ", " ", " "]
+ )
+ is None
+ )
+ # Test available_moves
+ assert TicTacToeEnv.available_moves(board) == [2, 3, 5, 6, 7]
+
+
+# --- Opponent Class Tests ---
+
+
+def test_opponent_move_and_minimax():
+ """Test opponent move and minimax algorithm directly
+ from Opponent class."""
+ opponent = Opponent(play_style="optimal")
+ board = ["X", "X", " ", " ", "O", " ", " ", " ", " "]
+ # The optimal move should be index 2 (blocking X's win).
+ assert opponent.select_move(board) == 2
+ score, move = opponent.minimax(board, is_maximizing=False)
+ assert score == -1
+ assert move == 2
+
+
+def test_opponent_random_move():
+ """Test that a random opponent move is within the available moves."""
+ opponent = Opponent(play_style="random")
+ board = ["X", "O", " ", " ", "X", " ", " ", " ", " "]
+ available = TicTacToeEnv.available_moves(board)
+ move = opponent.select_move(board)
+ assert move in available
+
+
+def test_opponent_no_moves():
+ """Test that when no moves are available, the opponent returns None."""
+ opponent = Opponent(play_style="optimal")
+ board = ["X", "O", "X", "X", "O", "O", "O", "X", "X"]
+ move = opponent.select_move(board)
+ assert move is None
+
+
+def test_opponent_optimal_winning_move():
+ """Test that the optimal opponent identifies and makes a winning move,
+ if available."""
+ opponent = Opponent(play_style="optimal")
+ # Set up a board where O can win immediately (row 0: two O's and an empty).
+ board = ["O", "O", " ", "X", "X", " ", " ", " ", " "]
+ move = opponent.select_move(board)
+ # The winning move is index 2.
+ assert move == 2
+
+
+def test_minimax_draw_scenario():
+ """Test that minimax returns a draw score for a board leading to a draw."""
+ opponent = Opponent(play_style="optimal")
+ # A nearly complete board with one move left that results in a draw.
+ board = ["X", "O", "X", "X", "O", "O", "O", "X", " "]
+ score, move = opponent.minimax(board, is_maximizing=True)
+ # Only one move is available; playing it leads to a draw.
+ assert score == 0
+ # The only available move should be index 8.
+ assert move == 8
+
+
+# --- Error Handling Tests ---
+
+
+async def test_step_errors():
+ """Test errors for stepping without setup, after done, or without reset."""
+ # Without setup
+ env = TicTacToeEnv()
+ with pytest.raises(RuntimeError, match="Environment not set up"):
+ await env.step(Action(llm_response="<Action>5</Action>"))
+
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+ env._episode_ended = True
+ with pytest.raises(RuntimeError, match="Episode has ended"):
+ await env.step(Action(llm_response="<Action>5</Action>"))
+ await env.close()
+
+ env = TicTacToeEnv()
+ await env.setup()
+ env._last_observation = None # Simulate no reset
+ with pytest.raises(RuntimeError, match="No current observation"):
+ await env.step(Action(llm_response="<Action>5</Action>"))
+ await env.close()
+
+
+# --- Full test ---
+async def test_full_game():
+ """Test a full game sequence with intermediate reward shaping."""
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+
+ moves = ["1", "2", "4"]
+ for move in moves:
+ obs, reward, done, info = await env.step(
+ Action(llm_response=f"<Action>{move}</Action>")
+ )
+ if done:
+ winner = env._state["winner"]
+ if winner == "X":
+ assert reward == 1.0
+ elif winner == "O":
+ assert reward == 0.0
+ elif winner == "draw":
+ assert reward == 0.5
+ else:
+ # NEW: Validate reward matches optimal X evaluation
+ expected = TicTacToeEnv.evaluate_position_for_x(
+ env._state["board"], is_x_turn=True
+ )
+ assert reward == expected
+ assert 0.0 <= reward <= 1.0
+ assert done is False
+
+ await env.close()