Skip to content

A ~9M parameter LLM that talks like a small fish.

Python

building onnx

Arman Hossain091fab2aa685
diff --git a/tools/export_onnx.py b/tools/export_onnx.py
new file mode 100644
index 0000000..a7dce7f
--- /dev/null
+++ b/tools/export_onnx.py
@@ -0,0 +1,158 @@
+"""
+Export GuppyLM to ONNX format for browser inference via onnxruntime-web.
+
+Exports to web/model.onnx (quantized uint8 by default) + web/tokenizer.json
+so the browser demo (web/index.html) can load them — works on GitHub Pages.
+
+Quantization shrinks the model from ~35 MB (float32) to ~9 MB (uint8) with
+negligible quality loss at this model size.
+
+Usage:
+ python tools/export_onnx.py # quantized (default)
+ python tools/export_onnx.py --no-quantize # keep float32
+ python tools/export_onnx.py --push # also upload to HF repo
+"""
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "guppylm"))
+
+
+def load_env():
+ env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env")
+ if os.path.exists(env_path):
+ with open(env_path) as f:
+ for line in f:
+ line = line.strip()
+ if line and not line.startswith("#") and "=" in line:
+ key, val = line.split("=", 1)
+ os.environ.setdefault(key.strip(), val.strip())
+
+
+def export_onnx(checkpoint_path, tokenizer_path, output_path, quantize=True, push=False):
+ import shutil
+
+ import torch
+ from config import GuppyConfig
+ from model import GuppyLM
+
+ # Load checkpoint
+ ckpt = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
+
+ # Resolve config
+ config_dir = os.path.dirname(os.path.abspath(checkpoint_path))
+ config_json = os.path.join(config_dir, "config.json")
+
+ if os.path.exists(config_json):
+ with open(config_json) as f:
+ cfg = json.load(f)
+ config = GuppyConfig(
+ vocab_size=cfg.get("vocab_size", 4096),
+ max_seq_len=cfg.get("max_position_embeddings", cfg.get("max_seq_len", 128)),
+ d_model=cfg.get("hidden_size", cfg.get("d_model", 384)),
+ n_layers=cfg.get("num_hidden_layers", cfg.get("n_layers", 6)),
+ n_heads=cfg.get("num_attention_heads", cfg.get("n_heads", 6)),
+ ffn_hidden=cfg.get("intermediate_size", cfg.get("ffn_hidden", 768)),
+ dropout=cfg.get("hidden_dropout_prob", cfg.get("dropout", 0.1)),
+ pad_id=cfg.get("pad_token_id", cfg.get("pad_id", 0)),
+ bos_id=cfg.get("bos_token_id", cfg.get("bos_id", 1)),
+ eos_id=cfg.get("eos_token_id", cfg.get("eos_id", 2)),
+ )
+ elif isinstance(ckpt, dict) and "config" in ckpt:
+ valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}
+ config = GuppyConfig(**{k: v for k, v in ckpt["config"].items() if k in valid_fields})
+ else:
+ config = GuppyConfig()
+
+ # Load model
+ state_dict = ckpt["model_state_dict"] if isinstance(ckpt, dict) and "model_state_dict" in ckpt else ckpt
+ model = GuppyLM(config)
+ model.load_state_dict({k: v for k, v in state_dict.items() if k in model.state_dict()})
+ model.eval()
+
+ total = sum(p.numel() for p in model.parameters())
+ print(f"GuppyLM: {total:,} params ({total/1e6:.1f}M)")
+
+ # Export to ONNX — only the forward pass (logits), no loss
+ dummy_input = torch.randint(0, config.vocab_size, (1, 32))
+
+ out_dir = os.path.dirname(output_path) or "."
+ os.makedirs(out_dir, exist_ok=True)
+
+ torch.onnx.export(
+ model,
+ (dummy_input,),
+ output_path,
+ input_names=["input_ids"],
+ output_names=["logits"],
+ dynamic_axes={
+ "input_ids": {0: "batch", 1: "seq_len"},
+ "logits": {0: "batch", 1: "seq_len"},
+ },
+ opset_version=17,
+ )
+
+ size_mb = os.path.getsize(output_path) / 1e6
+ print(f"Exported {output_path} ({size_mb:.1f} MB, float32)")
+
+ # Quantize to uint8 — ~4x smaller, negligible quality loss at 9M params
+ if quantize:
+ from onnxruntime.quantization import quantize_dynamic, QuantType
+
+ fp32_path = output_path.replace(".onnx", "_fp32.onnx")
+ os.rename(output_path, fp32_path)
+
+ quantize_dynamic(
+ fp32_path,
+ output_path,
+ weight_type=QuantType.QUInt8,
+ )
+
+ q_size_mb = os.path.getsize(output_path) / 1e6
+ print(f"Quantized {output_path} ({q_size_mb:.1f} MB, uint8)")
+ os.remove(fp32_path)
+
+ # Copy tokenizer alongside the ONNX model (needed for web/ GitHub Pages demo)
+ tok_dest = os.path.join(out_dir, "tokenizer.json")
+ if os.path.abspath(tokenizer_path) != os.path.abspath(tok_dest):
+ shutil.copy2(tokenizer_path, tok_dest)
+ print(f"Copied tokenizer to {tok_dest}")
+
+ # Push to HuggingFace
+ if push:
+ load_env()
+ token = os.environ.get("HF_TOKEN")
+ repo = os.environ.get("HF_REPO")
+ if not token or not repo:
+ print("Set HF_TOKEN and HF_REPO in .env to push. Skipping.")
+ return
+
+ from huggingface_hub import HfApi
+ api = HfApi(token=token)
+ api.upload_file(
+ path_or_fileobj=output_path,
+ path_in_repo="model.onnx",
+ repo_id=repo,
+ repo_type="model",
+ )
+ print(f"Uploaded model.onnx to https://huggingface.co/{repo}")
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Export GuppyLM to ONNX")
+ parser.add_argument("--checkpoint", default="checkpoints/best_model.pt")
+ parser.add_argument("--tokenizer", default="data/tokenizer.json")
+ parser.add_argument("--output", default="web/model.onnx")
+ parser.add_argument("--no-quantize", action="store_true", help="Skip uint8 quantization")
+ parser.add_argument("--push", action="store_true", help="Upload to HuggingFace repo")
+ args = parser.parse_args()
+
+ export_onnx(args.checkpoint, args.tokenizer, args.output,
+ quantize=not args.no_quantize, push=args.push)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/make_colab.py b/tools/make_colab.py
index f5aa4d8..47b3dd0 100644
--- a/tools/make_colab.py
+++ b/tools/make_colab.py
@@ -287,62 +287,87 @@ def build():
))
# ══════════════════════════════════════════════════════════════════
- # 7. UPLOAD TO HUGGINGFACE
+ # 7. EXPORT & UPLOAD TO HUGGINGFACE
# ══════════════════════════════════════════════════════════════════
cells.append(md(
- "## 7. Upload to HuggingFace\n"
+ "## 7. Export & Upload to HuggingFace\n"
+ "\n"
+ "Export the model in both PyTorch and ONNX (quantized uint8, ~9 MB) formats,\n"
+ "then upload everything to HuggingFace in one go.\n"
"\n"
- "Push the trained model to HuggingFace Hub.\n"
"Set your token and repo below."
))
cells.append(code(
+ "!pip install -q onnx onnxruntime onnxscript\n"
+ "\n"
"from huggingface_hub import HfApi, login\n"
- "import torch, json, os\n"
+ "import torch, json, os, shutil\n"
+ "from config import GuppyConfig\n"
+ "from model import GuppyLM\n"
"\n"
"HF_TOKEN = os.environ.get('HF_TOKEN', '') # Or paste your token here\n"
"HF_REPO = os.environ.get('HF_REPO', 'arman-bd/guppylm-9M') # Or change this\n"
"\n"
- "if not HF_TOKEN:\n"
- " print('No HF_TOKEN found. Set it as env var or paste above. Skipping.')\n"
- "else:\n"
+ "# Load checkpoint\n"
+ "ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n"
+ "cfg = ckpt['config']\n"
+ "os.makedirs('hf_export', exist_ok=True)\n"
+ "\n"
+ "# ── PyTorch format ──\n"
+ "torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n"
+ "\n"
+ "with open('hf_export/config.json', 'w') as f:\n"
+ " json.dump({\n"
+ " 'model_type': 'guppylm',\n"
+ " 'architectures': ['GuppyLM'],\n"
+ " 'vocab_size': cfg['vocab_size'],\n"
+ " 'max_position_embeddings': cfg['max_seq_len'],\n"
+ " 'hidden_size': cfg['d_model'],\n"
+ " 'num_hidden_layers': cfg['n_layers'],\n"
+ " 'num_attention_heads': cfg['n_heads'],\n"
+ " 'intermediate_size': cfg['ffn_hidden'],\n"
+ " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n"
+ " 'pad_token_id': cfg['pad_id'],\n"
+ " 'bos_token_id': cfg['bos_id'],\n"
+ " 'eos_token_id': cfg['eos_id'],\n"
+ " }, f, indent=2)\n"
+ "\n"
+ "shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n"
+ "print(f'pytorch_model.bin: {os.path.getsize(\"hf_export/pytorch_model.bin\")/1e6:.1f} MB')\n"
+ "\n"
+ "# ── ONNX format (quantized uint8) ──\n"
+ "valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}\n"
+ "config = GuppyConfig(**{k: v for k, v in cfg.items() if k in valid_fields})\n"
+ "model = GuppyLM(config)\n"
+ "model.load_state_dict(ckpt['model_state_dict'])\n"
+ "model.eval()\n"
+ "\n"
+ "dummy = torch.randint(0, config.vocab_size, (1, 32))\n"
+ "fp32_path = 'hf_export/model_fp32.onnx'\n"
+ "torch.onnx.export(\n"
+ " model, (dummy,), fp32_path,\n"
+ " input_names=['input_ids'], output_names=['logits'],\n"
+ " dynamic_axes={'input_ids': {0: 'batch', 1: 'seq_len'},\n"
+ " 'logits': {0: 'batch', 1: 'seq_len'}},\n"
+ " opset_version=17,\n"
+ ")\n"
+ "\n"
+ "from onnxruntime.quantization import quantize_dynamic, QuantType\n"
+ "quantize_dynamic(fp32_path, 'hf_export/model.onnx', weight_type=QuantType.QUInt8)\n"
+ "os.remove(fp32_path)\n"
+ "print(f'model.onnx: {os.path.getsize(\"hf_export/model.onnx\")/1e6:.1f} MB (uint8)')\n"
+ "\n"
+ "# ── Upload ──\n"
+ "if HF_TOKEN:\n"
" login(token=HF_TOKEN)\n"
" api = HfApi()\n"
" api.create_repo(HF_REPO, exist_ok=True)\n"
- "\n"
- " # Convert to HuggingFace standard format\n"
- " os.makedirs('hf_export', exist_ok=True)\n"
- "\n"
- " # pytorch_model.bin — state_dict only\n"
- " ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n"
- " torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n"
- "\n"
- " # config.json — model architecture\n"
- " cfg = ckpt['config']\n"
- " with open('hf_export/config.json', 'w') as f:\n"
- " json.dump({\n"
- " 'model_type': 'guppylm',\n"
- " 'architectures': ['GuppyLM'],\n"
- " 'vocab_size': cfg['vocab_size'],\n"
- " 'max_position_embeddings': cfg['max_seq_len'],\n"
- " 'hidden_size': cfg['d_model'],\n"
- " 'num_hidden_layers': cfg['n_layers'],\n"
- " 'num_attention_heads': cfg['n_heads'],\n"
- " 'intermediate_size': cfg['ffn_hidden'],\n"
- " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n"
- " 'pad_token_id': cfg['pad_id'],\n"
- " 'bos_token_id': cfg['bos_id'],\n"
- " 'eos_token_id': cfg['eos_id'],\n"
- " }, f, indent=2)\n"
- "\n"
- " # tokenizer.json — at root\n"
- " import shutil\n"
- " shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n"
- "\n"
- " # Upload\n"
" api.upload_folder(folder_path='hf_export', repo_id=HF_REPO, repo_type='model')\n"
- " print(f'Done! https://huggingface.co/{HF_REPO}')"
+ " print(f'Done! https://huggingface.co/{HF_REPO}')\n"
+ "else:\n"
+ " print('No HF_TOKEN — exported locally to hf_export/')"
))
# ══════════════════════════════════════════════════════════════════
@@ -364,7 +389,8 @@ def build():
" guppy/data/tokenizer.json \\\n"
" guppy/model.py \\\n"
" guppy/config.py \\\n"
- " guppy/inference.py\n"
+ " guppy/inference.py \\\n"
+ " guppy/hf_export/model.onnx\n"
"\n"
"sz = os.path.getsize('/content/guppylm.tar.gz') / 1e6\n"
"print(f'Package: /content/guppylm.tar.gz ({sz:.1f} MB)')\n"
diff --git a/train_guppylm.ipynb b/train_guppylm.ipynb
index ca2f32d..40176e4 100644
--- a/train_guppylm.ipynb
+++ b/train_guppylm.ipynb
@@ -830,9 +830,11 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## 7. Upload to HuggingFace\n",
+ "## 7. Export & Upload to HuggingFace\n",
+ "\n",
+ "Export the model in both PyTorch and ONNX (quantized uint8, ~9 MB) formats,\n",
+ "then upload everything to HuggingFace in one go.\n",
"\n",
- "Push the trained model to HuggingFace Hub.\n",
"Set your token and repo below."
]
},
@@ -840,51 +842,74 @@
"cell_type": "code",
"metadata": {},
"source": [
+ "!pip install -q onnx onnxruntime onnxscript\n",
+ "\n",
"from huggingface_hub import HfApi, login\n",
- "import torch, json, os\n",
+ "import torch, json, os, shutil\n",
+ "from config import GuppyConfig\n",
+ "from model import GuppyLM\n",
"\n",
"HF_TOKEN = os.environ.get('HF_TOKEN', '') # Or paste your token here\n",
"HF_REPO = os.environ.get('HF_REPO', 'arman-bd/guppylm-9M') # Or change this\n",
"\n",
- "if not HF_TOKEN:\n",
- " print('No HF_TOKEN found. Set it as env var or paste above. Skipping.')\n",
- "else:\n",
+ "# Load checkpoint\n",
+ "ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n",
+ "cfg = ckpt['config']\n",
+ "os.makedirs('hf_export', exist_ok=True)\n",
+ "\n",
+ "# \u2500\u2500 PyTorch format \u2500\u2500\n",
+ "torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n",
+ "\n",
+ "with open('hf_export/config.json', 'w') as f:\n",
+ " json.dump({\n",
+ " 'model_type': 'guppylm',\n",
+ " 'architectures': ['GuppyLM'],\n",
+ " 'vocab_size': cfg['vocab_size'],\n",
+ " 'max_position_embeddings': cfg['max_seq_len'],\n",
+ " 'hidden_size': cfg['d_model'],\n",
+ " 'num_hidden_layers': cfg['n_layers'],\n",
+ " 'num_attention_heads': cfg['n_heads'],\n",
+ " 'intermediate_size': cfg['ffn_hidden'],\n",
+ " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n",
+ " 'pad_token_id': cfg['pad_id'],\n",
+ " 'bos_token_id': cfg['bos_id'],\n",
+ " 'eos_token_id': cfg['eos_id'],\n",
+ " }, f, indent=2)\n",
+ "\n",
+ "shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n",
+ "print(f'pytorch_model.bin: {os.path.getsize(\"hf_export/pytorch_model.bin\")/1e6:.1f} MB')\n",
+ "\n",
+ "# \u2500\u2500 ONNX format (quantized uint8) \u2500\u2500\n",
+ "valid_fields = {f.name for f in GuppyConfig.__dataclass_fields__.values()}\n",
+ "config = GuppyConfig(**{k: v for k, v in cfg.items() if k in valid_fields})\n",
+ "model = GuppyLM(config)\n",
+ "model.load_state_dict(ckpt['model_state_dict'])\n",
+ "model.eval()\n",
+ "\n",
+ "dummy = torch.randint(0, config.vocab_size, (1, 32))\n",
+ "fp32_path = 'hf_export/model_fp32.onnx'\n",
+ "torch.onnx.export(\n",
+ " model, (dummy,), fp32_path,\n",
+ " input_names=['input_ids'], output_names=['logits'],\n",
+ " dynamic_axes={'input_ids': {0: 'batch', 1: 'seq_len'},\n",
+ " 'logits': {0: 'batch', 1: 'seq_len'}},\n",
+ " opset_version=17,\n",
+ ")\n",
+ "\n",
+ "from onnxruntime.quantization import quantize_dynamic, QuantType\n",
+ "quantize_dynamic(fp32_path, 'hf_export/model.onnx', weight_type=QuantType.QUInt8)\n",
+ "os.remove(fp32_path)\n",
+ "print(f'model.onnx: {os.path.getsize(\"hf_export/model.onnx\")/1e6:.1f} MB (uint8)')\n",
+ "\n",
+ "# \u2500\u2500 Upload \u2500\u2500\n",
+ "if HF_TOKEN:\n",
" login(token=HF_TOKEN)\n",
" api = HfApi()\n",
" api.create_repo(HF_REPO, exist_ok=True)\n",
- "\n",
- " # Convert to HuggingFace standard format\n",
- " os.makedirs('hf_export', exist_ok=True)\n",
- "\n",
- " # pytorch_model.bin \u2014 state_dict only\n",
- " ckpt = torch.load('checkpoints/best_model.pt', map_location='cpu', weights_only=False)\n",
- " torch.save(ckpt['model_state_dict'], 'hf_export/pytorch_model.bin')\n",
- "\n",
- " # config.json \u2014 model architecture\n",
- " cfg = ckpt['config']\n",
- " with open('hf_export/config.json', 'w') as f:\n",
- " json.dump({\n",
- " 'model_type': 'guppylm',\n",
- " 'architectures': ['GuppyLM'],\n",
- " 'vocab_size': cfg['vocab_size'],\n",
- " 'max_position_embeddings': cfg['max_seq_len'],\n",
- " 'hidden_size': cfg['d_model'],\n",
- " 'num_hidden_layers': cfg['n_layers'],\n",
- " 'num_attention_heads': cfg['n_heads'],\n",
- " 'intermediate_size': cfg['ffn_hidden'],\n",
- " 'hidden_dropout_prob': cfg.get('dropout', 0.1),\n",
- " 'pad_token_id': cfg['pad_id'],\n",
- " 'bos_token_id': cfg['bos_id'],\n",
- " 'eos_token_id': cfg['eos_id'],\n",
- " }, f, indent=2)\n",
- "\n",
- " # tokenizer.json \u2014 at root\n",
- " import shutil\n",
- " shutil.copy('data/tokenizer.json', 'hf_export/tokenizer.json')\n",
- "\n",
- " # Upload\n",
" api.upload_folder(folder_path='hf_export', repo_id=HF_REPO, repo_type='model')\n",
- " print(f'Done! https://huggingface.co/{HF_REPO}')"
+ " print(f'Done! https://huggingface.co/{HF_REPO}')\n",
+ "else:\n",
+ " print('No HF_TOKEN \u2014 exported locally to hf_export/')"
],
"outputs": [],
"execution_count": null
@@ -910,7 +935,8 @@
" guppy/data/tokenizer.json \\\n",
" guppy/model.py \\\n",
" guppy/config.py \\\n",
- " guppy/inference.py\n",
+ " guppy/inference.py \\\n",
+ " guppy/hf_export/model.onnx\n",
"\n",
"sz = os.path.getsize('/content/guppylm.tar.gz') / 1e6\n",
"print(f'Package: /content/guppylm.tar.gz ({sz:.1f} MB)')\n",