Create, Evaluate, and Connect AI Skills
Python ai-agentsai-skillsartificial-intelligenceknowledge-graphlarge-language-modelsnatural-language-processingopenkgskillnetskills
import os
from openai import OpenAI
import re
from retry import retry
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
import json
import argparse
import yaml
import alfworld
import alfworld.agents.environment
from alfworld.agents.environment import get_environment
import sys
from src.alfworld.prompts.system_prompt import alfworld_system_prompt
from src.skill import SkillModule
client = OpenAI(
api_key=os.environ["API_KEY"],
base_url=os.environ["BASE_URL"]
)
@retry(tries=5, delay=5, backoff=2, jitter=(1, 3))
def llm(prompt, model="YOUR_MODEL_NAME"):
print(f'Calling LLM with model: {model}')
# Normalize messages format
if isinstance(prompt, list):
messages = prompt
elif isinstance(prompt, str):
messages = [{"role": "user", "content": prompt}]
else:
raise ValueError(f'prompt must be a list or a string, but got {type(prompt)}')
response = client.chat.completions.create(
model=model,
messages=messages
)
# Extract content
content = response.choices[0].message.content
if content is not None:
return content
return "Output Error"
def process_ob(ob):
if ob.startswith('You arrive at loc '):
ob = ob[ob.find('. ')+2:]
return ob
# ==========================================
# Constants & Configuration
# ==========================================
class Colors:
RED = '\033[91m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
BLUE = '\033[94m'
RESET = '\033[0m'
# ==========================================
# Helper Functions
# ==========================================
def parse_action(response: str) -> str:
"""
Extracts the action from the LLM response using regex.
"""
pattern = re.compile(r"Action:\s*(.+)", re.IGNORECASE)
match = pattern.search(response)
if match:
return match.group(1).strip().strip('"\'*`')
return ""
def run_standard_procedure(env, llm, model, process_ob, messages, max_steps):
"""
Executes the standard interaction loop when no skill is available
or when falling back from a failed skill retrieval.
"""
task_done = False
task_reward = 0
current_steps = 0
while not task_done and current_steps < max_steps:
current_steps += 1
# 1. Get Agent Response
try:
response = llm(messages, model)
print(f'{Colors.GREEN}Agent response: \n{response}{Colors.RESET}')
except Exception as e:
print(f'{Colors.RED}Error in LLM call: {e}{Colors.RESET}')
break
messages.append({"role": "assistant", "content": response})
# 2. Parse and Execute Action
action = parse_action(response)
action_list = [action]
observation, task_reward, done, info = env.step(action_list)
# 3. Process Observation
observation, task_reward, task_done = (
process_ob(observation[0]),
info["won"][0],
done[0]
)
print(f'{Colors.YELLOW}Observation: \n{observation}{Colors.RESET}')
# 4. Update History
messages.append({"role": "user", "content": f"Observation: {observation}"})
if task_done:
print(f'{Colors.GREEN}Whole Task completed! Reward: {task_reward}{Colors.RESET}')
return messages, task_done, task_reward, current_steps
# ==========================================
# Core Execution Logic
# ==========================================
def alfworld_run_single(env, obs=[], names=[], max_steps=30, model=None, Skill_Module=None):
"""
Execute a batch of Alfworld tasks (typically size 1 when parallelized).
Handles both standard execution and skill-augmented procedure execution.
"""
results = []
for task_idx, (ob, name) in enumerate(zip(obs, names)):
print(f'{Colors.RED}Processing task {task_idx + 1}/{len(obs)}: {name}{Colors.RESET}')
# Extract specific task query
query = ob.split('Your task is to: ')[-1].split('\n')[0].strip()
# Initialize context
messages = [{"role": "system", "content": alfworld_system_prompt}]
messages.append({"role": "user", "content": ob})
# Initialize loop variables
task_done = False
task_reward = 0
steps = 0
relevant_skill_names = []
overall_procedure = ""
overall_procedure_code = ""
# Determine Execution Strategy (Skill_Module vs Standard)
use_skill = False
if Skill_Module is not None:
relevant_skill_names = Skill_Module.retrieve_relevant_skills(ob)
if relevant_skill_names:
use_skill = True
print(f'{Colors.BLUE}Retrieved relevant skills: {relevant_skill_names}{Colors.RESET}')
else:
print(f"[INFO] No relevant skills found for task: {name}. Falling back to standard execution.")
# Execute Strategy
if use_skill:
overall_procedure = Skill_Module.generate_overall_procedure(ob, relevant_skill_names)
print(f'\n{Colors.BLUE}Generated Overall Procedure:\n{overall_procedure}{Colors.RESET}')
MAX_RETRIES = 3
retries = 0
while retries < MAX_RETRIES:
try:
overall_procedure_code = Skill_Module.generate_overall_procedure_code(ob, overall_procedure)
print(f'\n{Colors.BLUE}Generated Overall Procedure Code:\n{overall_procedure_code}{Colors.RESET}')
# Dynamic execution of generated procedure
namespace = {}
exec(overall_procedure_code, namespace)
func = namespace["overall_procedure_code"]
messages, task_done, task_reward, steps = func(
env, llm, model, process_ob, parse_action, messages, max_steps
)
if task_done:
print(f'{Colors.GREEN} Task completed! Reward: {task_reward}{Colors.RESET}')
break
except Exception as e:
print(f'Error loading/executing procedure script: {e}')
retries += 1
else:
# Fallback to Standard Execution Loop
messages, task_done, task_reward, steps = run_standard_procedure(
env, llm, model, process_ob, messages, max_steps
)
# Record Results
results.append({
"query": query,
"name": name,
"task_done": task_done,
"reward": task_reward,
"steps": steps,
"messages": messages,
"relevant_skill_names": relevant_skill_names
})
return results
def eval_single_game(game_idx, args, config, split, output_path):
"""
Worker function: Initializes an independent environment, fast-forwards to the
specific task, and performs the evaluation.
"""
try:
# 1. Independent Environment Initialization (Batch Size = 1)
# This ensures thread safety/process isolation.
env = get_environment(config["env"]["type"])(config, train_eval=split)
env = env.init_env(batch_size=1)
# 2. Fast-forward to specific Game ID
obs_list = []
info = {}
for _ in range(game_idx + 1):
obs_list, info = env.reset()
# 3. Initialize SkillModule (if enabled)
Skill_Module = None
if args.use_skill:
skill_config = {
"skills_dir": "src/skills/alfworld",
"overall_procedure_examples_path": "src/alfworld/alfworld_overall_procedure_examples.txt",
"procedure_code_template_path": "src/alfworld/alfworld_procedure_code_template.py",
"model": args.model
}
Skill_Module = SkillModule(**skill_config)
# 4. Prepare Observation
ob_str = '\n'.join(obs_list[0].split('\n\n')[1:])
game_name = '/'.join(info['extra.gamefile'][0].split('/')[-3:-1])
# 5. Execute Task
batch_results = alfworld_run_single(
env=env,
obs=[ob_str],
names=[game_name],
max_steps=args.max_steps,
model=args.model,
Skill_Module=Skill_Module
)
result = batch_results[0]
# 6. Save Results (Atomic write per file)
save_file = f'{output_path}/idx_{game_idx}.json'
with open(save_file, 'w') as f:
json.dump(result, f, indent=4, ensure_ascii=False)
return result
except Exception as e:
print(f"Error in game {game_idx}: {e}")
return None
finally:
# 6. CRITICAL: Close the environment
if env:
env.close()
# ==========================================
# Main Entry Point
# ==========================================
def main(args):
model_name = args.model
# Load configuration
with open('src/alfworld/base_config.yaml') as reader:
config = yaml.safe_load(reader)
# Determine split
split = "eval_in_distribution" if args.split == 'dev' else "eval_out_of_distribution"
# Setup output directory
output_path = f'results/alfworld/{model_name}/{args.split}_{args.exp_name}_skill_{args.use_skill}'
if not os.path.exists(output_path):
os.makedirs(output_path, exist_ok=True)
# ---------------------------------------------------------
# Determine Total Game Count
# ---------------------------------------------------------
# Temporarily initialize environment to get the total number of games
temp_env = get_environment(config["env"]["type"])(config, train_eval=split)
temp_env = temp_env.init_env(batch_size=1)
num_games = len(temp_env.gamefiles)
del temp_env
print(f"Total games to evaluate: {num_games}")
# ---------------------------------------------------------
# Identify Remaining Tasks (Checkpointing)
# ---------------------------------------------------------
tasks_to_run = []
finished_games = 0
all_rewards = 0
all_steps = 0
existing_files = set()
# Scan existing results
if os.path.exists(output_path):
for file in os.listdir(output_path):
if file.endswith('.json') and file.startswith('idx_'):
try:
idx = int(file.split('_')[1].split('.')[0])
existing_files.add(idx)
with open(f'{output_path}/{file}', 'r') as f:
res = json.load(f)
all_rewards += res['reward']
all_steps += res['steps']
finished_games += 1
except:
continue
# Filter tasks to run
for idx in range(num_games):
if idx not in existing_files:
tasks_to_run.append(idx)
print(f"Already finished: {finished_games}, Remaining: {len(tasks_to_run)}")
# ---------------------------------------------------------
# Parallel Execution
# ---------------------------------------------------------
max_workers = args.max_workers
print(f"Starting parallel evaluation with {max_workers} workers...")
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
# Submit all tasks
future_to_idx = {
executor.submit(
eval_single_game,
idx,
args,
config,
split,
output_path
): idx for idx in tasks_to_run
}
# Monitor progress with tqdm
pbar = tqdm(total=len(tasks_to_run), desc="Evaluating ALFWorld")
for future in concurrent.futures.as_completed(future_to_idx):
idx = future_to_idx[future]
try:
result = future.result()
if result:
finished_games += 1
all_rewards += result['reward']
all_steps += result['steps']
# Update progress bar description
current_avg_reward = all_rewards / finished_games if finished_games > 0 else 0
current_avg_steps = all_steps / finished_games if finished_games > 0 else 0
pbar.set_postfix({
'Avg Reward': f'{current_avg_reward:.2f}',
'Avg Steps': f'{current_avg_steps:.2f}'
})
except Exception as exc:
print(f'\nGame {idx} generated an exception: {exc}')
pbar.update(1)
pbar.close()
# Final Summary
print(f"Evaluation finished. Total Games: {finished_games}")
if finished_games > 0:
print(f"Final Avg Reward: {all_rewards / finished_games}")
print(f"Final Avg Steps: {all_steps / finished_games}")
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, default='gpt-4o')
parser.add_argument('--split', type=str, default='dev')
parser.add_argument('--max_workers', type=int, default=10, help="Number of parallel workers")
parser.add_argument('--max_steps', type=int, default=30)
parser.add_argument('--exp_name', type=str, default='')
parser.add_argument('--use_skill', action='store_true')
args = parser.parse_args()
main(args)