The best ChatGPT that $100 can buy.
Python
delete the whole report thing i think it was a bad idea it just bloats the code. doing this subtracts 540 LOC
Andrej Karpathyf10bd751efb1
diff --git a/README.md b/README.mdindex 483f3e3..87a9381 100644--- a/README.md+++ b/README.md@@ -155,7 +155,6 @@ I've published a number of guides that might contain helpful information, most r│ ├── logo.svg│ ├── loss_eval.py # Evaluate bits per byte (instead of loss)│ ├── optim.py # AdamW + Muon optimizer, 1GPU and distributed-│ ├── report.py # Utilities for writing the nanochat Report│ ├── tokenizer.py # BPE Tokenizer wrapper in style of GPT-4│ └── ui.html # HTML/CSS/JS for nanochat frontend├── pyproject.tomldiff --git a/nanochat/report.py b/nanochat/report.pydeleted file mode 100644index 5e74b98..0000000--- a/nanochat/report.py+++ /dev/null@@ -1,418 +0,0 @@-"""-Utilities for generating training report cards. More messy code than usual, will fix.-"""--import os-import re-import shutil-import subprocess-import socket-import datetime-import platform-import psutil-import torch--def run_command(cmd):- """Run a shell command and return output, or None if it fails."""- try:- result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=5)- # Return stdout if we got output (even if some files in xargs failed)- if result.stdout.strip():- return result.stdout.strip()- if result.returncode == 0:- return ""- return None- except:- return None--def get_git_info():- """Get current git commit, branch, and dirty status."""- info = {}- info['commit'] = run_command("git rev-parse --short HEAD") or "unknown"- info['branch'] = run_command("git rev-parse --abbrev-ref HEAD") or "unknown"-- # Check if repo is dirty (has uncommitted changes)- status = run_command("git status --porcelain")- info['dirty'] = bool(status) if status is not None else False-- # Get commit message- info['message'] = run_command("git log -1 --pretty=%B") or ""- info['message'] = info['message'].split('\n')[0][:80] # First line, truncated-- return info--def get_gpu_info():- """Get GPU information."""- if not torch.cuda.is_available():- return {"available": False}-- num_devices = torch.cuda.device_count()- info = {- "available": True,- "count": num_devices,- "names": [],- "memory_gb": []- }-- for i in range(num_devices):- props = torch.cuda.get_device_properties(i)- info["names"].append(props.name)- info["memory_gb"].append(props.total_memory / (1024**3))-- # Get CUDA version- info["cuda_version"] = torch.version.cuda or "unknown"-- return info--def get_system_info():- """Get system information."""- info = {}-- # Basic system info- info['hostname'] = socket.gethostname()- info['platform'] = platform.system()- info['python_version'] = platform.python_version()- info['torch_version'] = torch.__version__-- # CPU and memory- info['cpu_count'] = psutil.cpu_count(logical=False)- info['cpu_count_logical'] = psutil.cpu_count(logical=True)- info['memory_gb'] = psutil.virtual_memory().total / (1024**3)-- # User and environment- info['user'] = os.environ.get('USER', 'unknown')- info['nanochat_base_dir'] = os.environ.get('NANOCHAT_BASE_DIR', 'out')- info['working_dir'] = os.getcwd()-- return info--def estimate_cost(gpu_info, runtime_hours=None):- """Estimate training cost based on GPU type and runtime."""-- # Rough pricing, from Lambda Cloud- default_rate = 2.0- gpu_hourly_rates = {- "H100": 3.00,- "A100": 1.79,- "V100": 0.55,- }-- if not gpu_info.get("available"):- return None-- # Try to identify GPU type from name- hourly_rate = None- gpu_name = gpu_info["names"][0] if gpu_info["names"] else "unknown"- for gpu_type, rate in gpu_hourly_rates.items():- if gpu_type in gpu_name:- hourly_rate = rate * gpu_info["count"]- break-- if hourly_rate is None:- hourly_rate = default_rate * gpu_info["count"] # Default estimate-- return {- "hourly_rate": hourly_rate,- "gpu_type": gpu_name,- "estimated_total": hourly_rate * runtime_hours if runtime_hours else None- }--def generate_header():- """Generate the header for a training report."""- timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")-- git_info = get_git_info()- gpu_info = get_gpu_info()- sys_info = get_system_info()- cost_info = estimate_cost(gpu_info)-- header = f"""# nanochat training report--Generated: {timestamp}--## Environment--### Git Information-- Branch: {git_info['branch']}-- Commit: {git_info['commit']} {"(dirty)" if git_info['dirty'] else "(clean)"}-- Message: {git_info['message']}--### Hardware-- Platform: {sys_info['platform']}-- CPUs: {sys_info['cpu_count']} cores ({sys_info['cpu_count_logical']} logical)-- Memory: {sys_info['memory_gb']:.1f} GB-"""-- if gpu_info.get("available"):- gpu_names = ", ".join(set(gpu_info["names"]))- total_vram = sum(gpu_info["memory_gb"])- header += f"""- GPUs: {gpu_info['count']}x {gpu_names}-- GPU Memory: {total_vram:.1f} GB total-- CUDA Version: {gpu_info['cuda_version']}-"""- else:- header += "- GPUs: None available\n"-- if cost_info and cost_info["hourly_rate"] > 0:- header += f"""- Hourly Rate: ${cost_info['hourly_rate']:.2f}/hour\n"""-- header += f"""-### Software-- Python: {sys_info['python_version']}-- PyTorch: {sys_info['torch_version']}--"""-- # bloat metrics: count lines/chars in git-tracked source files only- extensions = ['py', 'md', 'rs', 'html', 'toml', 'sh']- git_patterns = ' '.join(f"'*.{ext}'" for ext in extensions)- files_output = run_command(f"git ls-files -- {git_patterns}")- file_list = [f for f in (files_output or '').split('\n') if f]- num_files = len(file_list)- num_lines = 0- num_chars = 0- if num_files > 0:- wc_output = run_command(f"git ls-files -- {git_patterns} | xargs wc -lc 2>/dev/null")- if wc_output:- total_line = wc_output.strip().split('\n')[-1]- parts = total_line.split()- if len(parts) >= 2:- num_lines = int(parts[0])- num_chars = int(parts[1])- num_tokens = num_chars // 4 # assume approximately 4 chars per token-- # count dependencies via uv.lock- uv_lock_lines = 0- if os.path.exists('uv.lock'):- with open('uv.lock', 'r', encoding='utf-8') as f:- uv_lock_lines = len(f.readlines())-- header += f"""-### Bloat-- Characters: {num_chars:,}-- Lines: {num_lines:,}-- Files: {num_files:,}-- Tokens (approx): {num_tokens:,}-- Dependencies (uv.lock lines): {uv_lock_lines:,}--"""- return header--# -------------------------------------------------------------------------------def slugify(text):- """Slugify a text string."""- return text.lower().replace(" ", "-")--# the expected files and their order-EXPECTED_FILES = [- "tokenizer-training.md",- "tokenizer-evaluation.md",- "base-model-training.md",- "base-model-loss.md",- "base-model-evaluation.md",- "chat-sft.md",- "chat-evaluation-sft.md",- "chat-rl.md",- "chat-evaluation-rl.md",-]-# the metrics we're currently interested in-chat_metrics = ["ARC-Easy", "ARC-Challenge", "MMLU", "GSM8K", "HumanEval", "ChatCORE"]--def extract(section, keys):- """simple def to extract a single key from a section"""- if not isinstance(keys, list):- keys = [keys] # convenience- out = {}- for line in section.split("\n"):- for key in keys:- if key in line:- out[key] = line.split(":")[1].strip()- return out--def extract_timestamp(content, prefix):- """Extract timestamp from content with given prefix."""- for line in content.split('\n'):- if line.startswith(prefix):- time_str = line.split(":", 1)[1].strip()- try:- return datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S")- except:- pass- return None--class Report:- """Maintains a bunch of logs, generates a final markdown report."""-- def __init__(self, report_dir):- os.makedirs(report_dir, exist_ok=True)- self.report_dir = report_dir-- def log(self, section, data):- """Log a section of data to the report."""- slug = slugify(section)- file_name = f"{slug}.md"- file_path = os.path.join(self.report_dir, file_name)- with open(file_path, "w", encoding="utf-8") as f:- f.write(f"## {section}\n")- f.write(f"timestamp: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")- for item in data:- if not item:- # skip falsy values like None or empty dict etc.- continue- if isinstance(item, str):- # directly write the string- f.write(item)- else:- # render a dict- for k, v in item.items():- if isinstance(v, float):- vstr = f"{v:.4f}"- elif isinstance(v, int) and v >= 10000:- vstr = f"{v:,.0f}"- else:- vstr = str(v)- f.write(f"- {k}: {vstr}\n")- f.write("\n")- return file_path-- def generate(self):- """Generate the final report."""- report_dir = self.report_dir- report_file = os.path.join(report_dir, "report.md")- print(f"Generating report to {report_file}")- final_metrics = {} # the most important final metrics we'll add as table at the end- start_time = None- end_time = None- with open(report_file, "w", encoding="utf-8") as out_file:- # write the header first- header_file = os.path.join(report_dir, "header.md")- if os.path.exists(header_file):- with open(header_file, "r", encoding="utf-8") as f:- header_content = f.read()- out_file.write(header_content)- start_time = extract_timestamp(header_content, "Run started:")- # capture bloat data for summary later (the stuff after Bloat header and until \n\n)- bloat_data = re.search(r"### Bloat\n(.*?)\n\n", header_content, re.DOTALL)- bloat_data = bloat_data.group(1) if bloat_data else ""- else:- start_time = None # will cause us to not write the total wall clock time- bloat_data = "[bloat data missing]"- print(f"Warning: {header_file} does not exist. Did you forget to run `nanochat reset`?")- # process all the individual sections- for file_name in EXPECTED_FILES:- section_file = os.path.join(report_dir, file_name)- if not os.path.exists(section_file):- print(f"Warning: {section_file} does not exist, skipping")- continue- with open(section_file, "r", encoding="utf-8") as in_file:- section = in_file.read()- # Extract timestamp from this section (the last section's timestamp will "stick" as end_time)- if "rl" not in file_name:- # Skip RL sections for end_time calculation because RL is experimental- end_time = extract_timestamp(section, "timestamp:")- # extract the most important metrics from the sections- if file_name == "base-model-evaluation.md":- final_metrics["base"] = extract(section, "CORE")- if file_name == "chat-evaluation-sft.md":- final_metrics["sft"] = extract(section, chat_metrics)- if file_name == "chat-evaluation-rl.md":- final_metrics["rl"] = extract(section, "GSM8K") # RL only evals GSM8K- # append this section of the report- out_file.write(section)- out_file.write("\n")- # add the final metrics table- out_file.write("## Summary\n\n")- # Copy over the bloat metrics from the header- out_file.write(bloat_data)- out_file.write("\n\n")- # Collect all unique metric names- all_metrics = set()- for stage_metrics in final_metrics.values():- all_metrics.update(stage_metrics.keys())- # Custom ordering: CORE first, ChatCORE last, rest in middle- all_metrics = sorted(all_metrics, key=lambda x: (x != "CORE", x == "ChatCORE", x))- # Fixed column widths- stages = ["base", "sft", "rl"]- metric_width = 15- value_width = 8- # Write table header- header = f"| {'Metric'.ljust(metric_width)} |"- for stage in stages:- header += f" {stage.upper().ljust(value_width)} |"- out_file.write(header + "\n")- # Write separator- separator = f"|{'-' * (metric_width + 2)}|"- for stage in stages:- separator += f"{'-' * (value_width + 2)}|"- out_file.write(separator + "\n")- # Write table rows- for metric in all_metrics:- row = f"| {metric.ljust(metric_width)} |"- for stage in stages:- value = final_metrics.get(stage, {}).get(metric, "-")- row += f" {str(value).ljust(value_width)} |"- out_file.write(row + "\n")- out_file.write("\n")- # Calculate and write total wall clock time- if start_time and end_time:- duration = end_time - start_time- total_seconds = int(duration.total_seconds())- hours = total_seconds // 3600- minutes = (total_seconds % 3600) // 60- out_file.write(f"Total wall clock time: {hours}h{minutes}m\n")- else:- out_file.write("Total wall clock time: unknown\n")- # also cp the report.md file to current directory- print(f"Copying report.md to current directory for convenience")- shutil.copy(report_file, "report.md")- return report_file-- def reset(self):- """Reset the report."""- # Remove section files- for file_name in EXPECTED_FILES:- file_path = os.path.join(self.report_dir, file_name)- if os.path.exists(file_path):- os.remove(file_path)- # Remove report.md if it exists- report_file = os.path.join(self.report_dir, "report.md")- if os.path.exists(report_file):- os.remove(report_file)- # Generate and write the header section with start timestamp- header_file = os.path.join(self.report_dir, "header.md")- header = generate_header()- start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")- with open(header_file, "w", encoding="utf-8") as f:- f.write(header)- f.write(f"Run started: {start_time}\n\n---\n\n")- print(f"Reset report and wrote header to {header_file}")--# ------------------------------------------------------------------------------# nanochat-specific convenience functions--class DummyReport:- def log(self, *args, **kwargs):- pass- def reset(self, *args, **kwargs):- pass--def get_report():- # just for convenience, only rank 0 logs to report- from nanochat.common import get_base_dir, get_dist_info- ddp, ddp_rank, ddp_local_rank, ddp_world_size = get_dist_info()- if ddp_rank == 0:- report_dir = os.path.join(get_base_dir(), "report")- return Report(report_dir)- else:- return DummyReport()--if __name__ == "__main__":- import argparse- parser = argparse.ArgumentParser(description="Generate or reset nanochat training reports.")- parser.add_argument("command", nargs="?", default="generate", choices=["generate", "reset"], help="Operation to perform (default: generate)")- args = parser.parse_args()- if args.command == "generate":- get_report().generate()- elif args.command == "reset":- get_report().reset()diff --git a/runs/speedrun.sh b/runs/speedrun.shindex 48fcc68..5f5a4ba 100644--- a/runs/speedrun.sh+++ b/runs/speedrun.sh@@ -39,12 +39,6 @@ if [ -z "$WANDB_RUN" ]; thenWANDB_RUN=dummyfi-# ------------------------------------------------------------------------------# During the course of the run, we will be writing markdown reports to the report/-# directory in the base dir. This command clears it out and writes a header section-# with a bunch of system info and a timestamp that marks the start of the run.-python -m nanochat.report reset-# -----------------------------------------------------------------------------# Tokenizer@@ -90,8 +84,3 @@ torchrun --standalone --nproc_per_node=8 -m scripts.chat_eval -- -i sft# even better, chat with your model over a pretty WebUI ChatGPT style# python -m scripts.chat_web--# ------------------------------------------------------------------------------# Generate the full report by putting together all the sections-# report.md is the output and will be copied to current directory for convenience-python -m nanochat.report generatediff --git a/scripts/base_eval.py b/scripts/base_eval.pyindex a57bbaf..b814552 100644--- a/scripts/base_eval.py+++ b/scripts/base_eval.py@@ -297,25 +297,6 @@ def main():print0(f"\nResults written to: {output_csv_path}")print0(f"CORE metric: {core_results['core_metric']:.4f}")- # --- Log to report ---- from nanochat.report import get_report- report_data = [{"model": model_name}]-- if core_results:- report_data[0]["CORE metric"] = core_results["core_metric"]- report_data.append(core_results["centered_results"])-- if bpb_results:- report_data[0]["train bpb"] = bpb_results.get("train")- report_data[0]["val bpb"] = bpb_results.get("val")-- if samples:- report_data.append({f"sample {i}": s for i, s in enumerate(samples)})- if unconditioned_samples:- report_data.append({f"unconditioned {i}": s for i, s in enumerate(unconditioned_samples)})-- get_report().log(section="Base model evaluation", data=report_data)-compute_cleanup()diff --git a/scripts/base_train.py b/scripts/base_train.pyindex a161c47..d46e4f1 100644--- a/scripts/base_train.py+++ b/scripts/base_train.py@@ -599,32 +599,6 @@ print0(f"Total training time: {total_training_time/60:.2f}m")if val_bpb is not None:print0(f"Minimum validation bpb: {min_val_bpb:.6f}")-# Log to report-from nanochat.report import get_report-get_report().log(section="Base model training", data=[- user_config, # CLI args- { # stats about the training setup- "Number of parameters": num_params,- "Number of FLOPs per token": f"{num_flops_per_token:e}",- "Calculated number of iterations": num_iterations,- "Number of training tokens": total_tokens,- "Tokens : Scaling params ratio": total_batch_size * num_iterations / num_scaling_params,- "DDP world size": ddp_world_size,- "warmup_steps": args.warmup_steps,- "warmdown_ratio": args.warmdown_ratio,- "final_lr_frac": args.final_lr_frac,- },- { # stats about training outcomes- "Minimum validation bpb": min_val_bpb if val_bpb is not None else None,- "Final validation bpb": val_bpb,- "CORE metric estimate": results.get("core_metric", None),- "MFU %": f"{mfu:.2f}%",- "Total training flops": f"{flops_so_far:e}",- "Total training time": f"{total_training_time/60:.2f}m",- "Peak memory usage": f"{get_max_memory() / 1024 / 1024:.2f}MiB",- }-])-# cleanupwandb_run.finish() # wandb run finishcompute_cleanup()diff --git a/scripts/chat_eval.py b/scripts/chat_eval.pyindex 858d4c2..55692bb 100644--- a/scripts/chat_eval.py+++ b/scripts/chat_eval.py@@ -228,12 +228,9 @@ if __name__ == "__main__":results[task_name] = accprint0(f"{task_name} accuracy: {100 * acc:.2f}%")- # Log to report- from nanochat.report import get_report- all_tasks_were_evaluated = all(task_name in results for task_name in all_tasks)# calculate the ChatCORE metric if we can (similar to CORE, it's the mean centered accuracy)# this way, ChatCORE ranges from 0 (at random baseline) to 1 (peak performance)- chatcore_metric_dict = {}+ all_tasks_were_evaluated = all(task_name in results for task_name in all_tasks)if all_tasks_were_evaluated:centered_mean = 0for task_name, acc in results.items():@@ -241,11 +238,6 @@ if __name__ == "__main__":centered_acc = (acc - baseline_acc) / (1.0 - baseline_acc)centered_mean += centered_accchatcore_metric = centered_mean / len(results)- chatcore_metric_dict = {"ChatCORE metric": chatcore_metric}- get_report().log(section="Chat evaluation " + args.source, data=[- vars(args), # CLI args- results,- chatcore_metric_dict,- ])+ print0(f"ChatCORE metric: {chatcore_metric:.4f}")compute_cleanup()diff --git a/scripts/chat_rl.py b/scripts/chat_rl.pyindex cb2cb0e..91f17f5 100644--- a/scripts/chat_rl.py+++ b/scripts/chat_rl.py@@ -322,11 +322,5 @@ for step in range(num_steps):)print(f"✅ Saved model checkpoint to {checkpoint_dir}")-# Log to report-from nanochat.report import get_report-get_report().log(section="Chat RL", data=[- user_config, # CLI args-])-wandb_run.finish() # wandb run finishcompute_cleanup()diff --git a/scripts/chat_sft.py b/scripts/chat_sft.pyindex b46dd81..ff08fdc 100644--- a/scripts/chat_sft.py+++ b/scripts/chat_sft.py@@ -501,19 +501,6 @@ print0(f"Peak memory usage: {get_max_memory() / 1024 / 1024:.2f}MiB")print0(f"Total training time: {total_training_time/60:.2f}m")print0(f"Minimum validation bpb: {min_val_bpb:.4f}")-# Log to report-from nanochat.report import get_report-get_report().log(section="SFT", data=[- user_config, # CLI args- { # stats about the training setup- "Number of iterations": step,- "DDP world size": ddp_world_size,- },- { # stats about training outcomes- "Minimum validation bpb": min_val_bpb,- }-])-# cleanupwandb_run.finish() # wandb run finishcompute_cleanup()diff --git a/scripts/tok_eval.py b/scripts/tok_eval.pyindex 9233d71..be3538e 100644--- a/scripts/tok_eval.py+++ b/scripts/tok_eval.py@@ -241,25 +241,3 @@ def print_comparison(baseline_name, baseline_results, ours_results, all_text):# Print comparisonsprint_comparison("GPT-2", tokenizer_results['gpt2'], tokenizer_results['ours'], all_text)print_comparison("GPT-4", tokenizer_results['gpt4'], tokenizer_results['ours'], all_text)--# Log to report-from nanochat.report import get_report-lines = []-for baseline_name in ["GPT-2", "GPT-4"]:- baseline_key = baseline_name.lower().replace('-', '')- baseline_results = tokenizer_results[baseline_key]- ours_results = tokenizer_results['ours']- lines.append(f"### Comparison with {baseline_name}")- lines.append("")- lines.append("| Text Type | Bytes | " + baseline_name + " Tokens | " + baseline_name + " Ratio | Ours Tokens | Ours Ratio | Relative Diff % |")- lines.append("|-----------|-------|--------------|--------------|-------------|------------|-----------------|")- for name, text in all_text:- baseline_data = baseline_results[name]- ours_data = ours_results[name]- relative_diff = ((baseline_data['tokens'] - ours_data['tokens']) / baseline_data['tokens']) * 100- lines.append(f"| {name} | {baseline_data['bytes']} | {baseline_data['tokens']} | {baseline_data['ratio']:.2f} | {ours_data['tokens']} | {ours_data['ratio']:.2f} | {relative_diff:+.1f}% |")- lines.append("")-report_markdown = "\n".join(lines)-get_report().log(section="Tokenizer evaluation", data=[- report_markdown,-])diff --git a/scripts/tok_train.py b/scripts/tok_train.pyindex 90495b1..e08cd10 100644--- a/scripts/tok_train.py+++ b/scripts/tok_train.py@@ -89,18 +89,3 @@ token_bytes_path = os.path.join(tokenizer_dir, "token_bytes.pt")with open(token_bytes_path, "wb") as f:torch.save(token_bytes, f)print(f"Saved token_bytes to {token_bytes_path}")--# Log to report-from nanochat.report import get_report-token_bytes_nonzero = (token_bytes[token_bytes > 0]).to(dtype=torch.float32)-get_report().log(section="Tokenizer training", data=[- vars(args), # argparse command line arguments- {"train_time": train_time},- {"num_special_tokens": len(special_set)},- {- "token_bytes_min": int(token_bytes_nonzero.min().item()),- "token_bytes_max": int(token_bytes_nonzero.max().item()),- "token_bytes_mean": token_bytes_nonzero.mean().item(),- "token_bytes_std": token_bytes_nonzero.std().item(),- }-])