A ~9M parameter LLM that talks like a small fish.
Python
building onnx
Arman Hossain091fab2aa685
diff --git a/tools/export_onnx.py b/tools/export_onnx.pynew file mode 100644index 0000000..a7dce7f--- /dev/null+++ b/tools/export_onnx.py@@ -0,0 +1,158 @@+"""+Export GuppyLM to ONNX format for browser inference via onnxruntime-web.++Exports to web/model.onnx (quantized uint8 by default) + web/tokenizer.json+so the browser demo (web/index.html) can load them — works on GitHub Pages.++Quantization shrinks the model from ~35 MB (float32) to ~9 MB (uint8) with+negligible quality loss at this model size.++Usage:+ python tools/export_onnx.py # quantized (default)+ python tools/export_onnx.py --no-quantize # keep float32+ python tools/export_onnx.py --push # also upload to HF repo+"""++import argparse+import json+import os+import sys++sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "guppylm"))+++def load_env():+ env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env")+ if os.path.exists(env_path):+ with open(env_path) as f:+ for line in f:+ line = line.strip()+ if line and not line.startswith("#") and "=" in line:+ key, val = line.split("=", 1)+ os.environ.setdefault(key.strip(), val.strip())+++def export_onnx(checkpoint_path, tokenizer_path, output_path, quantize=True, push=False):+ import shutil++ import torch+ from config import GuppyConfig+ from model import GuppyLM++ # Load checkpoint+ ckpt = torch.load(checkpoint_path, map_location="cpu", weights_only=False)++ # Resolve config+ config_dir = os.path.dirname(os.path.abspath(checkpoint_path))+ config_json = os.path.join(config_dir, "config.json")++ if os.path.exists(config_json):+ with open(config_json) as f:+ cfg = json.load(f)+ config = GuppyConfig(+ vocab_size=cfg.get("vocab_size", 4096),+ max_seq_len=cfg.get("max_position_embeddings", cfg.get("max_seq_len", 128)),+ d_model=cfg.get("hidden_size", cfg.get("d_model", 384)),+ n_layers=cfg.get("num_hidden_layers", cfg.get("n_layers", 6)),+ n_heads=cfg.get("num_attention_heads", cfg.get("n_heads", 6)),+ ffn_hidden=cfg.get("intermediate_size", cfg.get("ffn_hidden", 768)),+ dropout=cfg.get("hidden_dropout_prob", cfg.get("dropout", 0.1)),+ pad_id=cfg.get("pad_token_id", cfg.get("pad_id", 0)),+ bos_id=cfg.get("bos_token_id", cfg.get("bos_id", 1)),+ eos_id=cfg.get("eos_token_id", cfg.get("eos_id", 2)),+ )+ elif isinstance(ckpt, dict) and "config" in ckpt:+ valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}+ config = GuppyConfig(**{k: v for k, v in ckpt["config"].items() if k in valid_fields})+ else:+ config = GuppyConfig()++ # Load model+ state_dict = ckpt["model_state_dict"] if isinstance(ckpt, dict) and "model_state_dict" in ckpt else ckpt+ model = GuppyLM(config)+ model.load_state_dict({k: v for k, v in state_dict.items() if k in model.state_dict()})+ model.eval()++ total = sum(p.numel() for p in model.parameters())+ print(f"GuppyLM: {total:,} params ({total/1e6:.1f}M)")++ # Export to ONNX — only the forward pass (logits), no loss+ dummy_input = torch.randint(0, config.vocab_size, (1, 32))++ out_dir = os.path.dirname(output_path) or "."+ os.makedirs(out_dir, exist_ok=True)++ torch.onnx.export(+ model,+ (dummy_input,),+ output_path,+ input_names=["input_ids"],+ output_names=["logits"],+ dynamic_axes={+ "input_ids": {0: "batch", 1: "seq_len"},+ "logits": {0: "batch", 1: "seq_len"},+ },+ opset_version=17,+ )++ size_mb = os.path.getsize(output_path) / 1e6+ print(f"Exported {output_path} ({size_mb:.1f} MB, float32)")++ # Quantize to uint8 — ~4x smaller, negligible quality loss at 9M params+ if quantize:+ from onnxruntime.quantization import quantize_dynamic, QuantType++ fp32_path = output_path.replace(".onnx", "_fp32.onnx")+ os.rename(output_path, fp32_path)++ quantize_dynamic(+ fp32_path,+ output_path,+ weight_type=QuantType.QUInt8,+ )++ q_size_mb = os.path.getsize(output_path) / 1e6+ print(f"Quantized {output_path} ({q_size_mb:.1f} MB, uint8)")+ os.remove(fp32_path)++ # Copy tokenizer alongside the ONNX model (needed for web/ GitHub Pages demo)+ tok_dest = os.path.join(out_dir, "tokenizer.json")+ if os.path.abspath(tokenizer_path) != os.path.abspath(tok_dest):+ shutil.copy2(tokenizer_path, tok_dest)+ print(f"Copied tokenizer to {tok_dest}")++ # Push to HuggingFace+ if push:+ load_env()+ token = os.environ.get("HF_TOKEN")+ repo = os.environ.get("HF_REPO")+ if not token or not repo:+ print("Set HF_TOKEN and HF_REPO in .env to push. Skipping.")+ return++ from huggingface_hub import HfApi+ api = HfApi(token=token)+ api.upload_file(+ path_or_fileobj=output_path,+ path_in_repo="model.onnx",+ repo_id=repo,+ repo_type="model",+ )+ print(f"Uploaded model.onnx to https://huggingface.co/{repo}")+++def main():+ parser = argparse.ArgumentParser(description="Export GuppyLM to ONNX")+ parser.add_argument("--checkpoint", default="checkpoints/best_model.pt")+ parser.add_argument("--tokenizer", default="data/tokenizer.json")+ parser.add_argument("--output", default="web/model.onnx")+ parser.add_argument("--no-quantize", action="store_true", help="Skip uint8 quantization")+ parser.add_argument("--push", action="store_true", help="Upload to HuggingFace repo")+ args = parser.parse_args()++ export_onnx(args.checkpoint, args.tokenizer, args.output,+ quantize=not args.no_quantize, push=args.push)+++if __name__ == "__main__":+ main()diff --git a/tools/make_colab.py b/tools/make_colab.pyindex f5aa4d8..47b3dd0 100644--- a/tools/make_colab.py+++ b/tools/make_colab.py@@ -287,62 +287,87 @@ def build():))# ══════════════════════════════════════════════════════════════════- # 7. UPLOAD TO HUGGINGFACE+ # 7. EXPORT & UPLOAD TO HUGGINGFACE# ══════════════════════════════════════════════════════════════════cells.append(md(- "## 7. Upload to HuggingFace\n"+ "## 7. Export & Upload to HuggingFace\n"+ "\n"+ "Export the model in both PyTorch and ONNX (quantized uint8, ~9 MB) formats,\n"+ "then upload everything to HuggingFace in one go.\n""\n"- "Push the trained model to HuggingFace Hub.\n""Set your token and repo below."))cells.append(code(+ "!pip install -q onnx onnxruntime onnxscript\n"+ "\n""from huggingface_hub import HfApi, login\n"- "import torch, json, os\n"+ "import torch, json, os, shutil\n"+ "from config import GuppyConfig\n"+ "from model import GuppyLM\n""\n""HF_TOKEN = os.environ.get('HF_TOKEN', '') # Or paste your token here\n""HF_REPO = os.environ.get('HF_REPO', 'arman-bd/guppylm-9M') # Or change this\n""\n"- "if not HF_TOKEN:\n"- " print('No HF_TOKEN found. Set it as env var or paste above. Skipping.')\n"- "else:\n"+ "# Load checkpoint\n"+ "ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n"+ "cfg = ckpt['config']\n"+ "os.makedirs('hf_export', exist_ok=True)\n"+ "\n"+ "# ── PyTorch format ──\n"+ "torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n"+ "\n"+ "with open('hf_export/config.json', 'w') as f:\n"+ " json.dump({\n"+ " 'model_type': 'guppylm',\n"+ " 'architectures': ['GuppyLM'],\n"+ " 'vocab_size': cfg['vocab_size'],\n"+ " 'max_position_embeddings': cfg['max_seq_len'],\n"+ " 'hidden_size': cfg['d_model'],\n"+ " 'num_hidden_layers': cfg['n_layers'],\n"+ " 'num_attention_heads': cfg['n_heads'],\n"+ " 'intermediate_size': cfg['ffn_hidden'],\n"+ " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n"+ " 'pad_token_id': cfg['pad_id'],\n"+ " 'bos_token_id': cfg['bos_id'],\n"+ " 'eos_token_id': cfg['eos_id'],\n"+ " }, f, indent=2)\n"+ "\n"+ "shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n"+ "print(f'pytorch_model.bin: {os.path.getsize(\"hf_export/pytorch_model.bin\")/1e6:.1f} MB')\n"+ "\n"+ "# ── ONNX format (quantized uint8) ──\n"+ "valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}\n"+ "config = GuppyConfig(**{k: v for k, v in cfg.items() if k in valid_fields})\n"+ "model = GuppyLM(config)\n"+ "model.load_state_dict(ckpt['model_state_dict'])\n"+ "model.eval()\n"+ "\n"+ "dummy = torch.randint(0, config.vocab_size, (1, 32))\n"+ "fp32_path = 'hf_export/model_fp32.onnx'\n"+ "torch.onnx.export(\n"+ " model, (dummy,), fp32_path,\n"+ " input_names=['input_ids'], output_names=['logits'],\n"+ " dynamic_axes={'input_ids': {0: 'batch', 1: 'seq_len'},\n"+ " 'logits': {0: 'batch', 1: 'seq_len'}},\n"+ " opset_version=17,\n"+ ")\n"+ "\n"+ "from onnxruntime.quantization import quantize_dynamic, QuantType\n"+ "quantize_dynamic(fp32_path, 'hf_export/model.onnx', weight_type=QuantType.QUInt8)\n"+ "os.remove(fp32_path)\n"+ "print(f'model.onnx: {os.path.getsize(\"hf_export/model.onnx\")/1e6:.1f} MB (uint8)')\n"+ "\n"+ "# ── Upload ──\n"+ "if HF_TOKEN:\n"" login(token=HF_TOKEN)\n"" api = HfApi()\n"" api.create_repo(HF_REPO, exist_ok=True)\n"- "\n"- " # Convert to HuggingFace standard format\n"- " os.makedirs('hf_export', exist_ok=True)\n"- "\n"- " # pytorch_model.bin — state_dict only\n"- " ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n"- " torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n"- "\n"- " # config.json — model architecture\n"- " cfg = ckpt['config']\n"- " with open('hf_export/config.json', 'w') as f:\n"- " json.dump({\n"- " 'model_type': 'guppylm',\n"- " 'architectures': ['GuppyLM'],\n"- " 'vocab_size': cfg['vocab_size'],\n"- " 'max_position_embeddings': cfg['max_seq_len'],\n"- " 'hidden_size': cfg['d_model'],\n"- " 'num_hidden_layers': cfg['n_layers'],\n"- " 'num_attention_heads': cfg['n_heads'],\n"- " 'intermediate_size': cfg['ffn_hidden'],\n"- " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n"- " 'pad_token_id': cfg['pad_id'],\n"- " 'bos_token_id': cfg['bos_id'],\n"- " 'eos_token_id': cfg['eos_id'],\n"- " }, f, indent=2)\n"- "\n"- " # tokenizer.json — at root\n"- " import shutil\n"- " shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n"- "\n"- " # Upload\n"" api.upload_folder(folder_path='hf_export', repo_id=HF_REPO, repo_type='model')\n"- " print(f'Done! https://huggingface.co/{HF_REPO}')"+ " print(f'Done! https://huggingface.co/{HF_REPO}')\n"+ "else:\n"+ " print('No HF_TOKEN — exported locally to hf_export/')"))# ══════════════════════════════════════════════════════════════════@@ -364,7 +389,8 @@ def build():" guppy/data/tokenizer.json \\\n"" guppy/model.py \\\n"" guppy/config.py \\\n"- " guppy/inference.py\n"+ " guppy/inference.py \\\n"+ " guppy/hf_export/model.onnx\n""\n""sz = os.path.getsize('/content/guppylm.tar.gz') / 1e6\n""print(f'Package: /content/guppylm.tar.gz ({sz:.1f} MB)')\n"diff --git a/train_guppylm.ipynb b/train_guppylm.ipynbindex ca2f32d..40176e4 100644--- a/train_guppylm.ipynb+++ b/train_guppylm.ipynb@@ -830,9 +830,11 @@"cell_type": "markdown","metadata": {},"source": [- "## 7. Upload to HuggingFace\n",+ "## 7. Export & Upload to HuggingFace\n",+ "\n",+ "Export the model in both PyTorch and ONNX (quantized uint8, ~9 MB) formats,\n",+ "then upload everything to HuggingFace in one go.\n","\n",- "Push the trained model to HuggingFace Hub.\n","Set your token and repo below."]},@@ -840,51 +842,74 @@"cell_type": "code","metadata": {},"source": [+ "!pip install -q onnx onnxruntime onnxscript\n",+ "\n","from huggingface_hub import HfApi, login\n",- "import torch, json, os\n",+ "import torch, json, os, shutil\n",+ "from config import GuppyConfig\n",+ "from model import GuppyLM\n","\n","HF_TOKEN = os.environ.get('HF_TOKEN', '') # Or paste your token here\n","HF_REPO = os.environ.get('HF_REPO', 'arman-bd/guppylm-9M') # Or change this\n","\n",- "if not HF_TOKEN:\n",- " print('No HF_TOKEN found. Set it as env var or paste above. Skipping.')\n",- "else:\n",+ "# Load checkpoint\n",+ "ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n",+ "cfg = ckpt['config']\n",+ "os.makedirs('hf_export', exist_ok=True)\n",+ "\n",+ "# \u2500\u2500 PyTorch format \u2500\u2500\n",+ "torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n",+ "\n",+ "with open('hf_export/config.json', 'w') as f:\n",+ " json.dump({\n",+ " 'model_type': 'guppylm',\n",+ " 'architectures': ['GuppyLM'],\n",+ " 'vocab_size': cfg['vocab_size'],\n",+ " 'max_position_embeddings': cfg['max_seq_len'],\n",+ " 'hidden_size': cfg['d_model'],\n",+ " 'num_hidden_layers': cfg['n_layers'],\n",+ " 'num_attention_heads': cfg['n_heads'],\n",+ " 'intermediate_size': cfg['ffn_hidden'],\n",+ " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n",+ " 'pad_token_id': cfg['pad_id'],\n",+ " 'bos_token_id': cfg['bos_id'],\n",+ " 'eos_token_id': cfg['eos_id'],\n",+ " }, f, indent=2)\n",+ "\n",+ "shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n",+ "print(f'pytorch_model.bin: {os.path.getsize(\"hf_export/pytorch_model.bin\")/1e6:.1f} MB')\n",+ "\n",+ "# \u2500\u2500 ONNX format (quantized uint8) \u2500\u2500\n",+ "valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}\n",+ "config = GuppyConfig(**{k: v for k, v in cfg.items() if k in valid_fields})\n",+ "model = GuppyLM(config)\n",+ "model.load_state_dict(ckpt['model_state_dict'])\n",+ "model.eval()\n",+ "\n",+ "dummy = torch.randint(0, config.vocab_size, (1, 32))\n",+ "fp32_path = 'hf_export/model_fp32.onnx'\n",+ "torch.onnx.export(\n",+ " model, (dummy,), fp32_path,\n",+ " input_names=['input_ids'], output_names=['logits'],\n",+ " dynamic_axes={'input_ids': {0: 'batch', 1: 'seq_len'},\n",+ " 'logits': {0: 'batch', 1: 'seq_len'}},\n",+ " opset_version=17,\n",+ ")\n",+ "\n",+ "from onnxruntime.quantization import quantize_dynamic, QuantType\n",+ "quantize_dynamic(fp32_path, 'hf_export/model.onnx', weight_type=QuantType.QUInt8)\n",+ "os.remove(fp32_path)\n",+ "print(f'model.onnx: {os.path.getsize(\"hf_export/model.onnx\")/1e6:.1f} MB (uint8)')\n",+ "\n",+ "# \u2500\u2500 Upload \u2500\u2500\n",+ "if HF_TOKEN:\n"," login(token=HF_TOKEN)\n"," api = HfApi()\n"," api.create_repo(HF_REPO, exist_ok=True)\n",- "\n",- " # Convert to HuggingFace standard format\n",- " os.makedirs('hf_export', exist_ok=True)\n",- "\n",- " # pytorch_model.bin \u2014 state_dict only\n",- " ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n",- " torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n",- "\n",- " # config.json \u2014 model architecture\n",- " cfg = ckpt['config']\n",- " with open('hf_export/config.json', 'w') as f:\n",- " json.dump({\n",- " 'model_type': 'guppylm',\n",- " 'architectures': ['GuppyLM'],\n",- " 'vocab_size': cfg['vocab_size'],\n",- " 'max_position_embeddings': cfg['max_seq_len'],\n",- " 'hidden_size': cfg['d_model'],\n",- " 'num_hidden_layers': cfg['n_layers'],\n",- " 'num_attention_heads': cfg['n_heads'],\n",- " 'intermediate_size': cfg['ffn_hidden'],\n",- " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n",- " 'pad_token_id': cfg['pad_id'],\n",- " 'bos_token_id': cfg['bos_id'],\n",- " 'eos_token_id': cfg['eos_id'],\n",- " }, f, indent=2)\n",- "\n",- " # tokenizer.json \u2014 at root\n",- " import shutil\n",- " shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n",- "\n",- " # Upload\n"," api.upload_folder(folder_path='hf_export', repo_id=HF_REPO, repo_type='model')\n",- " print(f'Done! https://huggingface.co/{HF_REPO}')"+ " print(f'Done! https://huggingface.co/{HF_REPO}')\n",+ "else:\n",+ " print('No HF_TOKEN \u2014 exported locally to hf_export/')"],"outputs": [],"execution_count": null@@ -910,7 +935,8 @@" guppy/data/tokenizer.json \\\n"," guppy/model.py \\\n"," guppy/config.py \\\n",- " guppy/inference.py\n",+ " guppy/inference.py \\\n",+ " guppy/hf_export/model.onnx\n","\n","sz = os.path.getsize('/content/guppylm.tar.gz') / 1e6\n","print(f'Package: /content/guppylm.tar.gz ({sz:.1f} MB)')\n",