The best ChatGPT that $100 can buy.
Python
delete pandas dep in base_eval use csv instead
Andrej Karpathy7d2c4a3d957b
diff --git a/scripts/base_eval.py b/scripts/base_eval.pyindex 8efde4f..c488c8a 100644--- a/scripts/base_eval.py+++ b/scripts/base_eval.py@@ -1,5 +1,5 @@"""-Evlauate the CORE metric for a given model.+Evaluate the CORE metric for a given model.Run on a single GPU:python base_eval.py@@ -10,14 +10,13 @@ torchrun --nproc_per_node=8 base_eval.pyThe script will print the CORE metric to the console."""import os-import sys+import csvimport timeimport jsonimport randomimport yamlfrom contextlib import nullcontext-import pandas as pdimport torchfrom nanochat.common import compute_init, compute_cleanup, print0, get_base_dir, autodetect_device_type@@ -26,13 +25,12 @@ from nanochat.checkpoint_manager import load_modelfrom nanochat.core_eval import evaluate_task# ------------------------------------------------------------------------------# nanoChat specific function dealing with I/O etc.+# nanochat specific function dealing with I/O etc.def evaluate_model(model, tokenizer, device, max_per_task=-1):"""Evaluate a base model on the CORE benchmark.- max_per_task: crop the data to this many examples per task for testing (-1 = disable)- TODO: clean up this function, delete the need for all the files, for pandas dependency, etc."""# Load config and task metadatabase_dir = get_base_dir()@@ -43,7 +41,15 @@ def evaluate_model(model, tokenizer, device, max_per_task=-1):with open(config_path, 'r') as f:config = yaml.safe_load(f)tasks = config['icl_tasks']- eval_metadata = pd.read_csv(eval_meta_data)++ # Load random baseline values from eval metadata+ random_baselines = {}+ with open(eval_meta_data, 'r', encoding='utf-8') as f:+ reader = csv.DictReader(f)+ for row in reader:+ task_name = row['Eval Task']+ random_baseline = row['Random baseline']+ random_baselines[task_name] = float(random_baseline)# Evaluate each taskresults = {}@@ -75,8 +81,7 @@ def evaluate_model(model, tokenizer, device, max_per_task=-1):accuracy = evaluate_task(model, tokenizer, data, device, task_meta)results[label] = accuracy- row = eval_metadata[eval_metadata["Eval Task"] == label]- random_baseline = row["Random baseline"].values[0]+ random_baseline = random_baselines[label]centered_result = (accuracy - 0.01 * random_baseline) / (1.0 - 0.01 * random_baseline)centered_results[label] = centered_resultend_time = time.time()